From c47a8d0fb49c65bc744f5cf3717bcaccb2133e03 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:29:15 +0200 Subject: [PATCH 01/33] Port dependency updater tests (#3864) --- .../provision.bicep | 2 +- .../LongestBuildPathUpdater.cs | 6 +- .../LongestBuildPathUpdaterConfiguration.cs | 3 +- ...tionService.LongestBuildPathUpdater.csproj | 7 + .../Program.cs | 2 +- .../SubscriptionTriggerer.cs | 26 ++- .../SubscriptionTriggererConfiguration.cs | 26 ++- .../DependencyRegistrationTests.cs | 8 +- .../GlobalUsings.cs | 3 +- .../LongestBuildPathUpdaterTests.cs | 172 ++++++++++++++++ ...rvice.LongestBuildPathUpdater.Tests.csproj | 2 +- .../DependencyRegistrationTests.cs | 6 +- .../GlobalUsings.cs | 5 + .../SubscriptionTriggererTests.cs | 193 ++++++++++++++++++ 14 files changed, 436 insertions(+), 25 deletions(-) create mode 100644 test/ProductConstructionService.LongestBuildPathUpdater.Tests/LongestBuildPathUpdaterTests.cs create mode 100644 test/ProductConstructionService.SubscriptionTriggerer.Tests/GlobalUsings.cs create mode 100644 test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 95b2017316..309c165cee 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -418,7 +418,7 @@ var acrPushRole = subscriptionResourceId('Microsoft.Authorization/roleDefinition var kvSecretUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '4633458b-17de-408a-b874-0445c86b69e6') // azure system role for setting storage queue access var storageQueueContrubutorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '974c5e8b-45b9-4653-ba55-5f855dd0fb88') -// azure system role for setting controbutor access +// azure system role for setting contributor access var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'b24988ac-6180-42a0-ab88-20f7382dd24c') // azure system role Key Vault Reader var keyVaultReaderRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '21090545-7ca7-4776-b22c-e363652d74d2') diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdater.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdater.cs index 0cb921146f..f55c1a5c52 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdater.cs @@ -61,8 +61,8 @@ public async Task UpdateLongestBuildPathAsync() channel.Name, lbp.BestCaseTimeInMinutes, lbp.WorstCaseTimeInMinutes); - // TODO Don't actually do anythign in BAR before we migrate fully to PCS - // await _context.LongestBuildPaths.AddAsync(lbp); + // TODO (https://github.com/dotnet/arcade-services/issues/3808) Don't actually do anything in BAR before we migrate fully to PCS + //await _context.LongestBuildPaths.AddAsync(lbp); } else { @@ -72,7 +72,7 @@ public async Task UpdateLongestBuildPathAsync() flowGraph.Nodes.Count); } } - // TODO Don't actually do anythign in BAR before we migrate fully to PCS + // TODO (https://github.com/dotnet/arcade-services/issues/3808) Don't actually do anything in BAR before we migrate fully to PCS //await _context.SaveChangesAsync(); } } diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs index 08a832dae4..d5c4015d1c 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs @@ -7,6 +7,7 @@ using ProductConstructionService.Common; namespace ProductConstructionService.LongestBuildPathUpdater; + public static class LongestBuildPathUpdaterConfiguration { public static void ConfigureLongestBuildPathUpdater( @@ -16,8 +17,6 @@ public static void ConfigureLongestBuildPathUpdater( builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); builder.AddBuildAssetRegistry(); - builder.Services.Configure(o => { }); - builder.Services.AddTransient(); } } diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj index a428e7a4fb..a80571b4b9 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj @@ -16,4 +16,11 @@ + + + + PreserveNewest + PreserveNewest + + diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs index 4b877f103b..810f68231b 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs @@ -43,7 +43,7 @@ await applicationScope.ServiceProvider.UseLocalWorkItemQueues( var triggerer = applicationScope.ServiceProvider.GetRequiredService(); - await triggerer.CheckSubscriptionsAsync(frequency); + await triggerer.TriggerSubscriptionsAsync(frequency); } finally { diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 08c22941f0..2cb801d212 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -26,8 +26,22 @@ public SubscriptionTriggerer( _workItemProducerFactory = workItemProducerFactory; } - public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) + public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) { + var workItemProducer = _workItemProducerFactory.CreateProducer(); + foreach (var updateSubscriptionWorkItem in await GetSubscriptionsToTrigger(targetUpdateFrequency)) + { + await workItemProducer.ProduceWorkItemAsync(updateSubscriptionWorkItem); + _logger.LogInformation("Queued update for subscription '{subscriptionId}' with build '{buildId}'", + updateSubscriptionWorkItem.SubscriptionId, + updateSubscriptionWorkItem.BuildId); + } + } + + private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) + { + List subscriptionsToTrigger = new(); + var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) .ToListAsync()) @@ -60,16 +74,14 @@ public async Task CheckSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) { - // TODO https://github.com/dotnet/arcade-services/issues/3811 add some kind of feature switch to trigger specific subscriptions - /*await _workItemProducerFactory.Create().ProduceWorkItemAsync(new() + subscriptionsToTrigger.Add(new UpdateSubscriptionWorkItem { BuildId = latestBuildInTargetChannel.Id, SubscriptionId = subscription.Id - });*/ - _logger.LogInformation("Queued update for subscription '{subscriptionId}' with build '{buildId}'", - subscription.Id, - latestBuildInTargetChannel.Id); + }); } } + + return subscriptionsToTrigger; } } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs index 8ec0b48ae5..253f5ed5e4 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs @@ -10,6 +10,7 @@ using ProductConstructionService.Common; using Azure.Identity; using ProductConstructionService.WorkItems; +using Azure.Storage.Queues.Models; namespace ProductConstructionService.SubscriptionTriggerer; @@ -28,9 +29,10 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); builder.AddBuildAssetRegistry(); - builder.AddWorkItemProducerFactory(credential); - - builder.Services.Configure(o => { }); + // TODO (https://github.com/dotnet/arcade-services/issues/3811) Use a fake WorkItemProducer untill we + // add some kind of feature switch to trigger specific subscriptions + //builder.AddWorkItemProducerFactory(credential); + builder.Services.AddTransient(); builder.Services.AddTransient(); builder.Services.AddTransient(sp => ActivatorUtilities.CreateInstance(sp, "git")); @@ -41,3 +43,21 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( return builder; } } + +internal class FakeWorkItemProducer : IWorkItemProducer where T : WorkItem +{ + public Task DeleteWorkItemAsync(string messageId, string popReceipt) + { + return Task.CompletedTask; + } + + public Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) + { + return Task.FromResult(QueuesModelFactory.SendReceipt("fake", DateTimeOffset.Now, DateTimeOffset.Now, "fake", DateTimeOffset.Now)); + } +} + +internal class FakeWorkItemProducerFacory : IWorkItemProducerFactory +{ + public IWorkItemProducer CreateProducer() where T : WorkItem => new FakeWorkItemProducer(); +} diff --git a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs index dc31fa1f79..8593111d21 100644 --- a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/DependencyRegistrationTests.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using FluentAssertions; using Microsoft.ApplicationInsights.Channel; using Microsoft.DotNet.Internal.DependencyInjection.Testing; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -10,7 +9,7 @@ namespace ProductConstructionService.LongestBuildPathUpdater.Tests; [TestFixture] -public class Tests +public class DependencyRegistrationTests { [Test] public void AreDependenciesRegistered() @@ -28,6 +27,9 @@ public void AreDependenciesRegistered() s.Add(descriptor); } }, - out var message).Should().BeTrue(message); + out var message, + additionalExemptTypes: [ + "Microsoft.Extensions.Hosting.ConsoleLifetimeOptions" + ]).Should().BeTrue(message); } } diff --git a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/GlobalUsings.cs b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/GlobalUsings.cs index ca8d412820..215c93316f 100644 --- a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/GlobalUsings.cs +++ b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/GlobalUsings.cs @@ -1,4 +1,5 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -global using NUnit.Framework; \ No newline at end of file +global using NUnit.Framework; +global using FluentAssertions; diff --git a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/LongestBuildPathUpdaterTests.cs b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/LongestBuildPathUpdaterTests.cs new file mode 100644 index 0000000000..5d8abde722 --- /dev/null +++ b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/LongestBuildPathUpdaterTests.cs @@ -0,0 +1,172 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Kusto; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; + +namespace ProductConstructionService.LongestBuildPathUpdater.Tests; + +[TestFixture] +[Ignore("TODO https://github.com/dotnet/arcade-services/issues/3808) this test will fail because the part where we write to BAR is commented out, the tests do work tho")] +public class LongestBuildPathUpdaterTests +{ + private BuildAssetRegistryContext? _context; + private ServiceProvider? _provider; + private IServiceScope _scope = new Mock().Object; + private Mock _barMock = new(); + + [SetUp] + public void Setup() + { + var services = new ServiceCollection(); + + _barMock = new Mock(); + services.AddLogging(); + services.AddDbContext( + options => + { + options.UseInMemoryDatabase("BuildAssetRegistry"); + options.EnableServiceProviderCaching(false); + }); + services.AddSingleton(new Mock().Object); + services.AddSingleton(_barMock.Object); + services.AddSingleton(new Mock().Object); + services.AddSingleton(_ => new Mock().Object); + + _provider = services.BuildServiceProvider(); + _scope = _provider.CreateScope(); + + _context = _scope.ServiceProvider.GetRequiredService(); + } + + [TearDown] + public void TearDown() + { + _scope.Dispose(); + _provider!.Dispose(); + } + + [Test] + public async Task ShouldSaveCorrectBestAndWorstPathTimes() + { + var graph1 = CreateGraph( + (Repo: "a", BestCaseTime: 1, WorstCaseTime: 7, OnLongestBuildPath: true), + (Repo: "b", BestCaseTime: 2, WorstCaseTime: 5, OnLongestBuildPath: true), + (Repo: "c", BestCaseTime: 9, WorstCaseTime: 9, OnLongestBuildPath: false)); + + var graph2 = CreateGraph( + (Repo: "g", BestCaseTime: 10, WorstCaseTime: 70, OnLongestBuildPath: true), + (Repo: "h", BestCaseTime: 20, WorstCaseTime: 50, OnLongestBuildPath: true)); + + SetupBar( + (ChannelId: 1, Graph: graph1), + (ChannelId: 2, Graph: graph2)); + + var updater = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await updater.UpdateLongestBuildPathAsync(); + + var longestBuildPaths = _context!.LongestBuildPaths.ToList(); + longestBuildPaths.Should().HaveCount(2); + + var firstChannelData = longestBuildPaths.FirstOrDefault(x => x.ChannelId == 1); + firstChannelData.Should().BeEquivalentTo(new LongestBuildPath + { + BestCaseTimeInMinutes = 2, + ChannelId = 1, + WorstCaseTimeInMinutes = 7, + ContributingRepositories = "b@main;a@main" + }, options => options + .Excluding(x => x.Channel) + .Excluding(x => x.Id) + .Excluding(x => x.ReportDate)); + + var secondChannelData = longestBuildPaths.FirstOrDefault(x => x.ChannelId == 2); + secondChannelData.Should().BeEquivalentTo(new LongestBuildPath + { + BestCaseTimeInMinutes = 20, + ChannelId = 2, + WorstCaseTimeInMinutes = 70, + ContributingRepositories = "h@main;g@main" + }, options => options + .Excluding(x => x.Channel) + .Excluding(x => x.Id) + .Excluding(x => x.ReportDate)); + } + + [Test] + public async Task ShouldNotAddLongestBuildPathRowWhenThereAreNoNodesOnLongestBuildPath() + { + var graph = CreateGraph( + (Repo: "a", BestCaseTime: 1, WorstCaseTime: 7, OnLongestBuildPath: false), + (Repo: "b", BestCaseTime: 2, WorstCaseTime: 5, OnLongestBuildPath: false)); + + SetupBar((ChannelId: 1, Graph: graph)); + + var updater = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await updater.UpdateLongestBuildPathAsync(); + + var longestBuildPaths = _context!.LongestBuildPaths.ToList(); + longestBuildPaths.Should().BeEmpty(); + } + + [Test] + public async Task ShouldNotAddLongestBuildPathRowWhenGraphIsEmpty() + { + var graph = new DependencyFlowGraph([], []); + + SetupBar((ChannelId: 1, Graph: graph)); + + var updater = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await updater.UpdateLongestBuildPathAsync(); + + var longestBuildPaths = _context!.LongestBuildPaths.ToList(); + longestBuildPaths.Should().BeEmpty(); + } + + private static DependencyFlowGraph CreateGraph( + params (string Repo, double BestCaseTime, double WorstCaseTime, bool OnLongestBuildPath)[] nodes) + { + var graphNodes = nodes + .Select((n, i) => new DependencyFlowNode(n.Repo, "main", $"Node{i}") + { + BestCasePathTime = n.BestCaseTime, + OnLongestBuildPath = n.OnLongestBuildPath, + WorstCasePathTime = n.WorstCaseTime + }) + .ToList(); + + return new DependencyFlowGraph(graphNodes, []); + } + + private void SetupBar( + params (int ChannelId, DependencyFlowGraph Graph)[] graphPerChannel) + { + foreach (var item in graphPerChannel) + { + _barMock + .Setup(m => m.GetDependencyFlowGraphAsync( + item.ChannelId, + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .ReturnsAsync(item.Graph); + + _context!.Channels.Add(new Channel + { + Id = item.ChannelId, + Name = $"Channel_{item.ChannelId}", + Classification = "Pizza", + }); + } + + _context!.SaveChanges(); + } +} diff --git a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/ProductConstructionService.LongestBuildPathUpdater.Tests.csproj b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/ProductConstructionService.LongestBuildPathUpdater.Tests.csproj index 8b2732ba69..325fbe3a1f 100644 --- a/test/ProductConstructionService.LongestBuildPathUpdater.Tests/ProductConstructionService.LongestBuildPathUpdater.Tests.csproj +++ b/test/ProductConstructionService.LongestBuildPathUpdater.Tests/ProductConstructionService.LongestBuildPathUpdater.Tests.csproj @@ -4,13 +4,13 @@ net8.0 enable enable - false true + diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs index 8d61fc55c0..643afd8f2e 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/DependencyRegistrationTests.cs @@ -1,12 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using FluentAssertions; + using Microsoft.ApplicationInsights.Channel; using Microsoft.DotNet.Internal.DependencyInjection.Testing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; -using NUnit.Framework; using ProductConstructionService.SubscriptionTriggerer; namespace SubscriptionTriggerer.Tests; @@ -34,7 +33,8 @@ public void AreDependenciesRegistered() }, out var message, additionalExemptTypes: [ - "Microsoft.Extensions.Azure.AzureClientsGlobalOptions" + "Microsoft.Extensions.Azure.AzureClientsGlobalOptions", + "Microsoft.Extensions.Hosting.ConsoleLifetimeOptions" ]) .Should().BeTrue(message); } diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/GlobalUsings.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/GlobalUsings.cs new file mode 100644 index 0000000000..a3ffe318ff --- /dev/null +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +global using NUnit.Framework; +global using FluentAssertions; diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs new file mode 100644 index 0000000000..cb09210749 --- /dev/null +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs @@ -0,0 +1,193 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Storage.Queues.Models; +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Kusto; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.SubscriptionTriggerer.Tests; + +[TestFixture] +public class SubscriptionTriggererTests +{ + private BuildAssetRegistryContext? _context; + private ServiceProvider? _provider; + private IServiceScope _scope = new Mock().Object; + private List _updateSubscriptionWorkItems = new(); + + [SetUp] + public void Setup() + { + var services = new ServiceCollection(); + + _updateSubscriptionWorkItems = new(); + Mock workItemProducerFactoryMock = new(); + Mock> workItemProducerMock = new(); + + workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) + .ReturnsAsync(QueuesModelFactory.SendReceipt("message", DateTimeOffset.Now, DateTimeOffset.Now, "popReceipt", DateTimeOffset.Now)) + .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); + workItemProducerFactoryMock.Setup(w => w.CreateProducer()) + .Returns(workItemProducerMock.Object); + + services.AddLogging(); + services.AddDbContext( + options => + { + options.UseInMemoryDatabase("BuildAssetRegistry"); + options.EnableServiceProviderCaching(false); + }); + services.AddSingleton(new Mock().Object); + services.AddSingleton(new Mock().Object); + services.AddSingleton(new Mock().Object); + services.AddSingleton(workItemProducerFactoryMock.Object); + services.AddSingleton(_ => new Mock().Object); + + _provider = services.BuildServiceProvider(); + _scope = _provider.CreateScope(); + + _context = _scope.ServiceProvider.GetRequiredService(); + } + + [TearDown] + public void TearDown() + { + _scope.Dispose(); + _provider!.Dispose(); + } + + [Test] + public async Task ShouldTriggerSubscription() + { + Channel channel = GetChannel(); + Build oldBuild = GetOldBuild(); + Build build = GetNewBuild(); + BuildChannel buildChannel = new() + { + Build = build, + Channel = channel + }; + Subscription subscription = GetSubscription(channel, oldBuild, true); + await _context!.Subscriptions.AddAsync(subscription); + await _context!.BuildChannels.AddAsync(buildChannel); + await _context!.SaveChangesAsync(); + + var triggerer = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await triggerer.TriggerSubscriptionsAsync(UpdateFrequency.EveryDay); + + _updateSubscriptionWorkItems.Count.Should().Be(1); + + var item = _updateSubscriptionWorkItems[0]; + + item.BuildId.Should().Be(build.Id); + item.SubscriptionId.Should().Be(subscription.Id); + } + + [Test] + public async Task ShouldNotUpdateSubscriptionBecauseNotEnabled() + { + var channel = GetChannel(); + var oldBuild = GetOldBuild(); + var build = GetNewBuild(); + var buildChannel = new BuildChannel + { + Build = build, + Channel = channel + }; + var subscription = GetSubscription(channel, oldBuild, false); + await _context!.Subscriptions.AddAsync(subscription); + await _context!.BuildChannels.AddAsync(buildChannel); + await _context!.SaveChangesAsync(); + + var triggerer = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await triggerer.TriggerSubscriptionsAsync(UpdateFrequency.EveryDay); + + _updateSubscriptionWorkItems.Count.Should().Be(0); + } + + [Test] + public async Task ShouldOnlyTriggerSubscriptionsWithCorrectUpdateFrequency() + { + Channel channel = GetChannel(); + Build oldBuild = GetOldBuild(); + Build build = GetNewBuild(); + BuildChannel buildChannel = new() + { + Build = build, + Channel = channel + }; + Subscription subscription = GetSubscription(channel, oldBuild, true); + await _context!.Subscriptions.AddAsync(subscription); + await _context!.BuildChannels.AddAsync(buildChannel); + await _context!.SaveChangesAsync(); + + var triggerer = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await triggerer.TriggerSubscriptionsAsync(UpdateFrequency.EveryWeek); + + _updateSubscriptionWorkItems.Count.Should().Be(0); + } + + [Test] + public async Task ShouldNotTriggerUpToDateSubscription() + { + var channel = GetChannel(); + var build = GetNewBuild(); + var buildChannel = new BuildChannel + { + Build = build, + Channel = channel + }; + var subscription = GetSubscription(channel, build, true); + await _context!.Subscriptions.AddAsync(subscription); + await _context!.BuildChannels.AddAsync(buildChannel); + await _context!.SaveChangesAsync(); + + var triggerer = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + await triggerer.TriggerSubscriptionsAsync(UpdateFrequency.EveryDay); + + _updateSubscriptionWorkItems.Count.Should().Be(0); + } + + private const string RepoName = "source.repo"; + + private Channel GetChannel() => new() + { + Name = "channel", + Classification = "class" + }; + + private Build GetOldBuild() => new() + { + AzureDevOpsRepository = RepoName, + DateProduced = DateTimeOffset.UtcNow.AddDays(-2) + }; + + private Build GetNewBuild() => new() + { + AzureDevOpsRepository = RepoName, + DateProduced = DateTimeOffset.UtcNow, + }; + + private Subscription GetSubscription(Channel channel, Build build, bool enabled) => new() + { + Channel = channel, + SourceRepository = RepoName, + TargetRepository = "target.repo", + TargetBranch = "target.branch", + Enabled = enabled, + PolicyObject = new SubscriptionPolicy + { + MergePolicies = null, + UpdateFrequency = UpdateFrequency.EveryDay + }, + LastAppliedBuild = build + }; +} From da712250c8e8a38b5f44c361ce53bfa39d833b73 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:18:00 +0200 Subject: [PATCH 02/33] [main] Update dependencies from dotnet/arcade (#3879) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 +++++++++---------- eng/Versions.props | 10 ++++---- .../templates/steps/telemetry-start.yml | 2 +- global.json | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 764d2dcfc8..a7e9cfe037 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 4460b755f3c7c89e9660d9580ff79afc4218dd85 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 33959d5b5f..5e2ca7a825 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24413.2 - 8.0.0-beta.24413.2 - 8.0.0-beta.24413.2 - 8.0.0-beta.24413.2 - 8.0.0-beta.24413.2 + 8.0.0-beta.24421.4 + 8.0.0-beta.24421.4 + 8.0.0-beta.24421.4 + 8.0.0-beta.24421.4 + 8.0.0-beta.24421.4 17.4.1 1.1.0-beta.24376.1 1.1.0-beta.24376.1 diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b5..6abbcb33a6 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -8,7 +8,7 @@ parameters: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 + - task: AzureKeyVault@2 inputs: azureSubscription: 'HelixProd_KeyVault' KeyVaultName: HelixProdKV diff --git a/global.json b/global.json index 775b5d6bef..278a08fc6d 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24413.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24421.4" } } From 29fbaee57170684f51b8014c1071f285aad90f13 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 18:18:15 +0200 Subject: [PATCH 03/33] [main] Update dependencies from dotnet/dnceng (#3878) Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 4 ++-- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 55170eb0c9..9d2285ddbf 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dnceng.secretmanager": { - "version": "1.1.0-beta.24413.3", + "version": "1.1.0-beta.24420.1", "commands": [ "secret-manager" ] @@ -15,7 +15,7 @@ ] }, "microsoft.dnceng.configuration.bootstrap": { - "version": "1.1.0-beta.24413.3", + "version": "1.1.0-beta.24420.1", "commands": [ "bootstrap-dnceng-configuration" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a7e9cfe037..b32149d9a1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -115,13 +115,13 @@ https://github.com/dotnet/arcade 4460b755f3c7c89e9660d9580ff79afc4218dd85 - + https://github.com/dotnet/dnceng - a06a815e1a599e209b49b37667074e3504810e67 + a0d339b12f248ca5b42d3a53a49c6b09ef1db579 - + https://github.com/dotnet/dnceng - a06a815e1a599e209b49b37667074e3504810e67 + a0d339b12f248ca5b42d3a53a49c6b09ef1db579 diff --git a/eng/Versions.props b/eng/Versions.props index 5e2ca7a825..6aba592617 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -37,8 +37,8 @@ 1.1.0-beta.24376.1 1.1.0-beta.24376.1 1.1.0-beta.24376.1 - 1.1.0-beta.24413.3 - 1.1.0-beta.24413.3 + 1.1.0-beta.24420.1 + 1.1.0-beta.24420.1 From 75ae48d22d5a3e6bbc23bde43ff1d6d61f48c8f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Didyk <106967057+oleksandr-didyk@users.noreply.github.com> Date: Tue, 27 Aug 2024 09:27:44 +0200 Subject: [PATCH 04/33] Remove AzDO PAT from CI tests (#3877) --- .vault-config/maestrolocal.yaml | 10 ---------- .vault-config/shared/maestro-secrets.yaml | 10 ---------- eng/templates/jobs/e2e-tests.yml | 7 ++++++- eng/templates/stages/deploy.yaml | 1 - 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.vault-config/maestrolocal.yaml b/.vault-config/maestrolocal.yaml index dae4a9b0ef..f87f74e309 100644 --- a/.vault-config/maestrolocal.yaml +++ b/.vault-config/maestrolocal.yaml @@ -18,13 +18,3 @@ secrets: hasPrivateKey: true hasWebhookSecret: false hasOAuthSecret: true - - dn-bot-dnceng-build-rw-code-rw-release-rw: - type: azure-devops-access-token - parameters: - domainAccountName: dn-bot - domainAccountSecret: - location: helixkv - name: dn-bot-account-redmond - organizations: dnceng - scopes: build_execute code_write release_execute diff --git a/.vault-config/shared/maestro-secrets.yaml b/.vault-config/shared/maestro-secrets.yaml index 544b4c9f99..38dbe367aa 100644 --- a/.vault-config/shared/maestro-secrets.yaml +++ b/.vault-config/shared/maestro-secrets.yaml @@ -4,13 +4,3 @@ github: hasPrivateKey: true hasWebhookSecret: true hasOAuthSecret: true - -dn-bot-dnceng-build-rw-code-rw-release-rw: - type: azure-devops-access-token - parameters: - domainAccountName: dn-bot - domainAccountSecret: - location: helixkv - name: dn-bot-account-redmond - organizations: dnceng - scopes: build_execute code_write release_execute diff --git a/eng/templates/jobs/e2e-tests.yml b/eng/templates/jobs/e2e-tests.yml index a1aec97511..e5e2fea754 100644 --- a/eng/templates/jobs/e2e-tests.yml +++ b/eng/templates/jobs/e2e-tests.yml @@ -96,6 +96,11 @@ jobs: .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --password "$(GetAuthInfo.Token)" --bar-uri "$(GetAuthInfo.BarUri)" --debug displayName: Test BAR token auth + - template: /eng/common/templates-official/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: "ArcadeServicesInternal" + outputVariableName: "AzdoToken" + - task: DotNetCoreCLI@2 displayName: Run E2E tests inputs: @@ -115,7 +120,7 @@ jobs: MAESTRO_BASEURIS: ${{ variables.MaestroTestEndpoints }} MAESTRO_TOKEN: $(GetAuthInfo.Token) GITHUB_TOKEN: $(maestro-scenario-test-github-token) - AZDO_TOKEN: $(dn-bot-dnceng-build-rw-code-rw-release-rw) + AZDO_TOKEN: $(AzdoToken) DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts DARC_DIR: $(Build.SourcesDirectory)\darc DARC_IS_CI: true diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 1f9fe168d6..311812c1c8 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -3,7 +3,6 @@ parameters: type: boolean # --- Secret Variable group requirements --- -# dn-bot-dnceng-build-rw-code-rw-release-rw # maestro-scenario-test-github-token stages: From 23f724a08e0c0c51d6f57482c1b0f0e3bf19856e Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:31:20 +0200 Subject: [PATCH 05/33] PCS should only trigger PCS subscriptions, and Maestro Maestro subscriptions (#3881) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- .../DependencyUpdater/DependencyUpdater.cs | 18 ++++++-- src/Maestro/DependencyUpdater/Program.cs | 2 + .../Maestro.Data/SubscriptionIdGenerator.cs | 43 +++++++++++++++++++ .../Controllers/SubscriptionsController.cs | 12 +++++- .../Controllers/SubscriptionsController.cs | 9 ++-- .../Controllers/SubscriptionsController.cs | 7 ++- src/Maestro/Maestro.Web/Startup.cs | 3 ++ .../Controllers/SubscriptionsController.cs | 12 +++++- .../Controllers/SubscriptionsController.cs | 6 ++- .../Controllers/SubscriptionsController.cs | 6 ++- .../PcsStartup.cs | 11 +++-- .../SubscriptionTriggerer.cs | 7 ++- .../SubscriptionTriggererConfiguration.cs | 27 ++---------- .../DependencyUpdaterTests.cs | 1 + .../SubscriptionsController20200220Tests.cs | 14 +++--- .../SubscriptionsController20200220Tests.cs | 15 ++++--- .../SubscriptionTriggererTests.cs | 2 + 17 files changed, 140 insertions(+), 55 deletions(-) create mode 100644 src/Maestro/Maestro.Data/SubscriptionIdGenerator.cs diff --git a/src/Maestro/DependencyUpdater/DependencyUpdater.cs b/src/Maestro/DependencyUpdater/DependencyUpdater.cs index 4e4a76b63a..f34e74734b 100644 --- a/src/Maestro/DependencyUpdater/DependencyUpdater.cs +++ b/src/Maestro/DependencyUpdater/DependencyUpdater.cs @@ -42,6 +42,8 @@ public sealed class DependencyUpdater : IServiceImplementation, IDependencyUpdat private readonly ILogger _logger; private readonly BuildAssetRegistryContext _context; private readonly IActorProxyFactory _subscriptionActorFactory; + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + private readonly SubscriptionIdGenerator _subscriptionIdGenerator; public DependencyUpdater( IReliableStateManager stateManager, @@ -49,7 +51,8 @@ public DependencyUpdater( BuildAssetRegistryContext context, IBasicBarClient barClient, IActorProxyFactory subscriptionActorFactory, - OperationManager operations) + OperationManager operations, + SubscriptionIdGenerator subscriptionIdGenerator) { _operations = operations; _stateManager = stateManager; @@ -57,6 +60,7 @@ public DependencyUpdater( _context = context; _barClient = barClient; _subscriptionActorFactory = subscriptionActorFactory; + _subscriptionIdGenerator = subscriptionIdGenerator; } public async Task StartUpdateDependenciesAsync(int buildId, int channelId) @@ -330,10 +334,16 @@ where sub.Enabled private async Task UpdateSubscriptionAsync(Guid subscriptionId, int buildId) { + if (!_subscriptionIdGenerator.ShouldTriggerSubscription(subscriptionId)) + { + _logger.LogInformation("Skipping subscription '{subscriptionId}', Maestro won't trigger PCS subscriptions", subscriptionId); + return; + } + using (_operations.BeginOperation( - "Updating subscription '{subscriptionId}' with build '{buildId}'", - subscriptionId, - buildId)) + "Updating subscription '{subscriptionId}' with build '{buildId}'", + subscriptionId, + buildId)) { try { diff --git a/src/Maestro/DependencyUpdater/Program.cs b/src/Maestro/DependencyUpdater/Program.cs index 8c3793a118..70bec120a5 100644 --- a/src/Maestro/DependencyUpdater/Program.cs +++ b/src/Maestro/DependencyUpdater/Program.cs @@ -63,5 +63,7 @@ public static void Configure(IServiceCollection services) services.AddScoped(); services.AddTransient(); services.AddKustoClientProvider("Kusto"); + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + services.AddSingleton(sp => new(RunningService.Maestro)); } } diff --git a/src/Maestro/Maestro.Data/SubscriptionIdGenerator.cs b/src/Maestro/Maestro.Data/SubscriptionIdGenerator.cs new file mode 100644 index 0000000000..0741d6dc84 --- /dev/null +++ b/src/Maestro/Maestro.Data/SubscriptionIdGenerator.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Maestro.Data; + +public enum RunningService +{ + Maestro, + PCS +} + +public class SubscriptionIdGenerator(RunningService runningService) +{ + private readonly RunningService _runningService = runningService; + + public const string PcsSubscriptionIdPrefix = "00000000"; + + public Guid GenerateSubscriptionId() + { + if (_runningService == RunningService.PCS) + { + return Guid.Parse($"{PcsSubscriptionIdPrefix}{Guid.NewGuid().ToString().Substring(PcsSubscriptionIdPrefix.Length)}"); + } + + var guid = Guid.NewGuid(); + + if (guid.ToString().StartsWith(PcsSubscriptionIdPrefix)) + { + return GenerateSubscriptionId(); + } + + return guid; + } + + public bool ShouldTriggerSubscription(Guid subscriptionId) + { + bool startsWithPcsSubscriptionIdPrefix = subscriptionId.ToString().StartsWith(PcsSubscriptionIdPrefix); + bool runningServiceIsPCS = _runningService == RunningService.PCS; + return runningServiceIsPCS == startsWithPcsSubscriptionIdPrefix; + } +} diff --git a/src/Maestro/Maestro.Web/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/Maestro/Maestro.Web/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 65588aa472..65aa678637 100644 --- a/src/Maestro/Maestro.Web/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/Maestro/Maestro.Web/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -32,13 +32,16 @@ public class SubscriptionsController : Controller { private readonly BuildAssetRegistryContext _context; private readonly IBackgroundQueue _queue; + protected readonly SubscriptionIdGenerator _subscriptionIdGenerator; public SubscriptionsController( BuildAssetRegistryContext context, - IBackgroundQueue queue) + IBackgroundQueue queue, + SubscriptionIdGenerator subscriptionIdGenerator) { _context = context; _queue = queue; + _subscriptionIdGenerator = subscriptionIdGenerator; } /// @@ -119,6 +122,11 @@ public virtual async Task TriggerSubscription(Guid id, [FromQuery protected async Task TriggerSubscriptionCore(Guid id, int buildId) { + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + if (!_subscriptionIdGenerator.ShouldTriggerSubscription(id)) + { + return BadRequest("Maestro shouldn't trigger PCS subscriptions"); + } Data.Models.Subscription subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -467,6 +475,8 @@ public virtual async Task Create([FromBody, Required] Subscriptio Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); Data.Models.Subscription equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) diff --git a/src/Maestro/Maestro.Web/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/Maestro/Maestro.Web/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 6aa9325f99..0a09d828b4 100644 --- a/src/Maestro/Maestro.Web/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/Maestro/Maestro.Web/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -27,8 +27,9 @@ public class SubscriptionsController : v2018_07_16.Controllers.SubscriptionsCont public SubscriptionsController( BuildAssetRegistryContext context, - IBackgroundQueue queue) - : base(context, queue) + IBackgroundQueue queue, + SubscriptionIdGenerator subscriptionIdGenerator) + : base(context, queue, subscriptionIdGenerator) { _context = context; } @@ -274,7 +275,9 @@ public override async Task Create([FromBody, Required] Maestro.Ap Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; - + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); + // Check that we're not about add an existing subscription that is identical Data.Models.Subscription equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) diff --git a/src/Maestro/Maestro.Web/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/Maestro/Maestro.Web/Api/v2020_02_20/Controllers/SubscriptionsController.cs index ff98c98d27..da09451bb8 100644 --- a/src/Maestro/Maestro.Web/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/Maestro/Maestro.Web/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -33,8 +33,9 @@ public class SubscriptionsController : v2019_01_16.Controllers.SubscriptionsCont public SubscriptionsController( BuildAssetRegistryContext context, IBackgroundQueue queue, - IGitHubClientFactory gitHubClientFactory) - : base(context, queue) + IGitHubClientFactory gitHubClientFactory, + SubscriptionIdGenerator subscriptionIdGenerator) + : base(context, queue, subscriptionIdGenerator) { _context = context; _gitHubClientFactory = gitHubClientFactory; @@ -444,6 +445,8 @@ public async Task Create([FromBody, Required] SubscriptionData su Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); // Check that we're not about add an existing subscription that is identical Data.Models.Subscription equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); diff --git a/src/Maestro/Maestro.Web/Startup.cs b/src/Maestro/Maestro.Web/Startup.cs index 12babe7b03..45cd2ca194 100644 --- a/src/Maestro/Maestro.Web/Startup.cs +++ b/src/Maestro/Maestro.Web/Startup.cs @@ -270,6 +270,9 @@ public override void ConfigureServices(IServiceCollection services) // in such a way that will work with sizing. services.AddSingleton(); + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + services.AddSingleton(sp => new(RunningService.Maestro)); + services.AddTransient(sp => new ProcessManager( sp.GetRequiredService>(), diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index a03f079da2..ff806c758f 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -27,15 +27,19 @@ public class SubscriptionsController : ControllerBase private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly ILogger _logger; + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + protected readonly SubscriptionIdGenerator _subscriptionIdGenerator; public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, - ILogger logger) + ILogger logger, + SubscriptionIdGenerator subscriptionIdGenerator) { _context = context; _workItemProducerFactory = workItemProducerFactory; _logger = logger; + _subscriptionIdGenerator = subscriptionIdGenerator; } /// @@ -112,6 +116,11 @@ public virtual async Task TriggerSubscription(Guid id, [FromQuery protected async Task TriggerSubscriptionCore(Guid id, int buildId) { + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + if (!_subscriptionIdGenerator.ShouldTriggerSubscription(id)) + { + return BadRequest($"PCS can only trigger subscriptions which ids start with ${SubscriptionIdGenerator.PcsSubscriptionIdPrefix}"); + } Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -427,6 +436,7 @@ public virtual async Task Create([FromBody, Required] Subscriptio Maestro.Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 06d31fe134..e3390d19d0 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -25,8 +25,9 @@ public class SubscriptionsController : v2018_07_16.Controllers.SubscriptionsCont public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, - ILogger logger) - : base(context, workItemProducerFactory, logger) + ILogger logger, + SubscriptionIdGenerator subscriptionIdGenerator) + : base(context, workItemProducerFactory, logger, subscriptionIdGenerator) { _context = context; } @@ -272,6 +273,7 @@ public override async Task Create([FromBody, Required] Maestro.Ap Maestro.Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); // Check that we're not about add an existing subscription that is identical Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index fc4efabac8..c8efa2c0ac 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -30,8 +30,9 @@ public SubscriptionsController( BuildAssetRegistryContext context, IGitHubClientFactory gitHubClientFactory, IWorkItemProducerFactory workItemProducerFactory, - ILogger logger) - : base(context, workItemProducerFactory, logger) + ILogger logger, + SubscriptionIdGenerator subscriptionIdGenerator) + : base(context, workItemProducerFactory, logger, subscriptionIdGenerator) { _context = context; _gitHubClientFactory = gitHubClientFactory; @@ -441,6 +442,7 @@ public async Task Create([FromBody, Required] SubscriptionData su Maestro.Data.Models.Subscription subscriptionModel = subscription.ToDb(); subscriptionModel.Channel = channel; + subscriptionModel.Id = _subscriptionIdGenerator.GenerateSubscriptionId(); // Check that we're not about add an existing subscription that is identical Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index a0303c56f4..c5aab43e08 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -76,6 +76,7 @@ static PcsStartup() var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); var workItemProducer = context.GetService().CreateProducer(); + var subscriptionIdGenerator = context.GetService(); BuildChannel entity = entry.Entity; Build? build = context.Builds @@ -100,17 +101,18 @@ static PcsStartup() sub.ChannelId == entity.ChannelId && (sub.SourceRepository == entity.Build.GitHubRepository || sub.SourceDirectory == entity.Build.AzureDevOpsRepository) && JsonExtensions.JsonValue(sub.PolicyString, "lax $.UpdateFrequency") == ((int)UpdateFrequency.EveryBuild).ToString()) + // TODO (https://github.com/dotnet/arcade-services/issues/3880) + .Where(sub => subscriptionIdGenerator.ShouldTriggerSubscription(sub.Id)) .ToList(); - // TODO: https://github.com/dotnet/arcade-services/issues/3811 Add a feature switch to trigger specific subscriptions - /*foreach (Subscription subscription in subscriptionsToUpdate) + foreach (Subscription subscription in subscriptionsToUpdate) { workItemProducer.ProduceWorkItemAsync(new() { BuildId = entity.BuildId, SubscriptionId = subscription.Id }).GetAwaiter().GetResult(); - }*/ + } } else { @@ -157,6 +159,9 @@ internal static async Task ConfigurePcs( builder.Configuration.AddAzureKeyVault(keyVaultUri, azureCredential); } + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + builder.Services.AddSingleton(sp => new(RunningService.PCS)); + builder.AddBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); builder.AddVmrRegistrations(gitHubToken); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 2cb801d212..1f1865e778 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -15,15 +15,18 @@ public class SubscriptionTriggerer private readonly ILogger _logger; private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; + private readonly SubscriptionIdGenerator _subscriptionIdGenerator; public SubscriptionTriggerer( ILogger logger, BuildAssetRegistryContext context, - IWorkItemProducerFactory workItemProducerFactory) + IWorkItemProducerFactory workItemProducerFactory, + SubscriptionIdGenerator subscriptionIdGenerator) { _logger = logger; _context = context; _workItemProducerFactory = workItemProducerFactory; + _subscriptionIdGenerator = subscriptionIdGenerator; } public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) @@ -44,6 +47,8 @@ private async Task> GetSubscriptionsToTrigger(U var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + .Where(s => _subscriptionIdGenerator.ShouldTriggerSubscription(s.Id)) .ToListAsync()) .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs index 253f5ed5e4..5778fa9f72 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs @@ -10,7 +10,7 @@ using ProductConstructionService.Common; using Azure.Identity; using ProductConstructionService.WorkItems; -using Azure.Storage.Queues.Models; +using Maestro.Data; namespace ProductConstructionService.SubscriptionTriggerer; @@ -29,10 +29,9 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); builder.AddBuildAssetRegistry(); - // TODO (https://github.com/dotnet/arcade-services/issues/3811) Use a fake WorkItemProducer untill we - // add some kind of feature switch to trigger specific subscriptions - //builder.AddWorkItemProducerFactory(credential); - builder.Services.AddTransient(); + builder.AddWorkItemProducerFactory(credential); + // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator + builder.Services.AddSingleton(sp => new(RunningService.PCS)); builder.Services.AddTransient(); builder.Services.AddTransient(sp => ActivatorUtilities.CreateInstance(sp, "git")); @@ -43,21 +42,3 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( return builder; } } - -internal class FakeWorkItemProducer : IWorkItemProducer where T : WorkItem -{ - public Task DeleteWorkItemAsync(string messageId, string popReceipt) - { - return Task.CompletedTask; - } - - public Task ProduceWorkItemAsync(T payload, TimeSpan delay = default) - { - return Task.FromResult(QueuesModelFactory.SendReceipt("fake", DateTimeOffset.Now, DateTimeOffset.Now, "fake", DateTimeOffset.Now)); - } -} - -internal class FakeWorkItemProducerFacory : IWorkItemProducerFactory -{ - public IWorkItemProducer CreateProducer() where T : WorkItem => new FakeWorkItemProducer(); -} diff --git a/test/DependencyUpdater.Tests/DependencyUpdaterTests.cs b/test/DependencyUpdater.Tests/DependencyUpdaterTests.cs index 754be5f54c..9745544a9c 100644 --- a/test/DependencyUpdater.Tests/DependencyUpdaterTests.cs +++ b/test/DependencyUpdater.Tests/DependencyUpdaterTests.cs @@ -60,6 +60,7 @@ public void DependencyUpdaterTests_SetUp() services.AddSingleton(BarMock.Object); services.AddTransient(_ => null); services.AddOperationTracking(o => { }); + services.AddSingleton(_ => new(RunningService.Maestro)); Provider = services.BuildServiceProvider(); Scope = Provider.CreateScope(); _context = new Lazy(GetContext); diff --git a/test/Maestro.Web.Tests/SubscriptionsController20200220Tests.cs b/test/Maestro.Web.Tests/SubscriptionsController20200220Tests.cs index 060e2a31e4..f93fe3bcdd 100644 --- a/test/Maestro.Web.Tests/SubscriptionsController20200220Tests.cs +++ b/test/Maestro.Web.Tests/SubscriptionsController20200220Tests.cs @@ -49,7 +49,8 @@ public async Task CreateGetAndListSubscriptions() string defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; string defaultAzdoSourceRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-source-repo"; string defaultAzdoTargetRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-target-repo"; - string defaultBranchName = "main"; + string branchName1 = "a"; + string branchName2 = "b"; string aValidDependencyFlowNotificationList = "@someMicrosoftUser;@some-github-team"; // Create two subscriptions @@ -60,7 +61,7 @@ public async Task CreateGetAndListSubscriptions() SourceRepository = defaultGitHubSourceRepo, TargetRepository = defaultGitHubTargetRepo, Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, - TargetBranch = defaultBranchName, + TargetBranch = branchName1, PullRequestFailureNotificationTags = aValidDependencyFlowNotificationList }; @@ -75,7 +76,7 @@ public async Task CreateGetAndListSubscriptions() createdSubscription1.Channel.Name.Should().Be(testChannelName); createdSubscription1.Policy.Batchable.Should().Be(true); createdSubscription1.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek); - createdSubscription1.TargetBranch.Should().Be(defaultBranchName); + createdSubscription1.TargetBranch.Should().Be(branchName1); createdSubscription1.SourceRepository.Should().Be(defaultGitHubSourceRepo); createdSubscription1.TargetRepository.Should().Be(defaultGitHubTargetRepo); createdSubscription1.PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); @@ -90,7 +91,7 @@ public async Task CreateGetAndListSubscriptions() SourceRepository = defaultAzdoSourceRepo, TargetRepository = defaultAzdoTargetRepo, Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = false, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.None }, - TargetBranch = defaultBranchName, + TargetBranch = branchName2, SourceEnabled = true, SourceDirectory = "sub-controller-test-source-repo", ExcludedAssets = [DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"], @@ -107,7 +108,7 @@ public async Task CreateGetAndListSubscriptions() createdSubscription2.Channel.Name.Should().Be(testChannelName); createdSubscription2.Policy.Batchable.Should().Be(false); createdSubscription2.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.None); - createdSubscription2.TargetBranch.Should().Be(defaultBranchName); + createdSubscription2.TargetBranch.Should().Be(branchName2); createdSubscription2.SourceRepository.Should().Be(defaultAzdoSourceRepo); createdSubscription2.TargetRepository.Should().Be(defaultAzdoTargetRepo); createdSubscription2.PullRequestFailureNotificationTags.Should().BeNull(); @@ -122,7 +123,7 @@ public async Task CreateGetAndListSubscriptions() result.Should().BeAssignableTo(); var objResult = (ObjectResult) result; objResult.StatusCode.Should().Be((int) HttpStatusCode.OK); - List listedSubs = ((IEnumerable) objResult.Value).ToList(); + List listedSubs = ((IEnumerable) objResult.Value).OrderBy(sub => sub.TargetBranch).ToList(); listedSubs.Count.Should().Be(2); listedSubs[0].Enabled.Should().Be(true); listedSubs[0].TargetRepository.Should().Be(defaultGitHubTargetRepo); @@ -463,6 +464,7 @@ public static void Dependencies(IServiceCollection collection) collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); collection.AddSingleton(); + collection.AddSingleton(_ => new SubscriptionIdGenerator(RunningService.Maestro)); } public static void GitHub(IServiceCollection collection) diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs index 919775cf48..a786d7b6e5 100644 --- a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -19,7 +19,6 @@ using Moq; using ProductConstructionService.Api.Api.v2020_02_20.Controllers; using ProductConstructionService.WorkItems; -using ProductConstructionService.Api.VirtualMonoRepo; using ProductConstructionService.DependencyFlow.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -47,7 +46,8 @@ public async Task CreateGetAndListSubscriptions() var defaultGitHubTargetRepo = "https://github.com/dotnet/sub-controller-test-target-repo"; var defaultAzdoSourceRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-source-repo"; var defaultAzdoTargetRepo = "https://dev.azure.com/dnceng/internal/_git/sub-controller-test-target-repo"; - var defaultBranchName = "main"; + var branchName1 = "a"; + var branchName2 = "b"; var aValidDependencyFlowNotificationList = "@someMicrosoftUser;@some-github-team"; // Create two subscriptions @@ -58,7 +58,7 @@ public async Task CreateGetAndListSubscriptions() SourceRepository = defaultGitHubSourceRepo, TargetRepository = defaultGitHubTargetRepo, Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = true, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek }, - TargetBranch = defaultBranchName, + TargetBranch = branchName1, PullRequestFailureNotificationTags = aValidDependencyFlowNotificationList }; @@ -73,7 +73,7 @@ public async Task CreateGetAndListSubscriptions() createdSubscription1.Channel.Name.Should().Be(testChannelName); createdSubscription1.Policy.Batchable.Should().Be(true); createdSubscription1.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.EveryWeek); - createdSubscription1.TargetBranch.Should().Be(defaultBranchName); + createdSubscription1.TargetBranch.Should().Be(branchName1); createdSubscription1.SourceRepository.Should().Be(defaultGitHubSourceRepo); createdSubscription1.TargetRepository.Should().Be(defaultGitHubTargetRepo); createdSubscription1.PullRequestFailureNotificationTags.Should().Be(aValidDependencyFlowNotificationList); @@ -88,7 +88,7 @@ public async Task CreateGetAndListSubscriptions() SourceRepository = defaultAzdoSourceRepo, TargetRepository = defaultAzdoTargetRepo, Policy = new Maestro.Api.Model.v2018_07_16.SubscriptionPolicy() { Batchable = false, UpdateFrequency = Maestro.Api.Model.v2018_07_16.UpdateFrequency.None }, - TargetBranch = defaultBranchName, + TargetBranch = branchName2, SourceEnabled = true, SourceDirectory = "sub-controller-test-source-repo", ExcludedAssets = [DependencyFileManager.ArcadeSdkPackageName, "Foo.Bar"], @@ -105,7 +105,7 @@ public async Task CreateGetAndListSubscriptions() createdSubscription2.Channel.Name.Should().Be(testChannelName); createdSubscription2.Policy.Batchable.Should().Be(false); createdSubscription2.Policy.UpdateFrequency.Should().Be(Maestro.Api.Model.v2018_07_16.UpdateFrequency.None); - createdSubscription2.TargetBranch.Should().Be(defaultBranchName); + createdSubscription2.TargetBranch.Should().Be(branchName2); createdSubscription2.SourceRepository.Should().Be(defaultAzdoSourceRepo); createdSubscription2.TargetRepository.Should().Be(defaultAzdoTargetRepo); createdSubscription2.PullRequestFailureNotificationTags.Should().BeNull(); @@ -120,7 +120,7 @@ public async Task CreateGetAndListSubscriptions() result.Should().BeAssignableTo(); var objResult = (ObjectResult)result; objResult.StatusCode.Should().Be((int)HttpStatusCode.OK); - var listedSubs = ((IEnumerable)objResult.Value!).ToList(); + var listedSubs = ((IEnumerable)objResult.Value!).OrderBy(sub => sub.TargetBranch).ToList(); listedSubs.Count.Should().Be(2); listedSubs[0].Enabled.Should().Be(true); listedSubs[0].TargetRepository.Should().Be(defaultGitHubTargetRepo); @@ -466,6 +466,7 @@ public static void Dependencies(IServiceCollection collection) collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); collection.AddSingleton(mockWorkItemProducerFactory.Object); + collection.AddSingleton(_ => new SubscriptionIdGenerator(RunningService.PCS)); } public static void GitHub(IServiceCollection collection) diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs index cb09210749..3edb477fa2 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs @@ -50,6 +50,7 @@ public void Setup() services.AddSingleton(new Mock().Object); services.AddSingleton(workItemProducerFactoryMock.Object); services.AddSingleton(_ => new Mock().Object); + services.AddSingleton(_ => new SubscriptionIdGenerator(RunningService.PCS)); _provider = services.BuildServiceProvider(); _scope = _provider.CreateScope(); @@ -178,6 +179,7 @@ public async Task ShouldNotTriggerUpToDateSubscription() private Subscription GetSubscription(Channel channel, Build build, bool enabled) => new() { + Id = _scope.ServiceProvider.GetRequiredService().GenerateSubscriptionId(), Channel = channel, SourceRepository = RepoName, TargetRepository = "target.repo", From 5200f8b23efffcd784e05d5f7e33c19f0619e234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Tue, 27 Aug 2024 16:47:34 +0200 Subject: [PATCH 06/33] Port non-actor dependency flow code (#3882) --- arcade-services.sln | 15 + .../Controllers/SubscriptionsController.cs | 4 +- .../Controllers/BuildsController.cs | 73 --- .../PcsStartup.cs | 5 +- .../ActorFactory.cs | 35 ++ .../BatchedPullRequestActor.cs | 37 ++ .../CodeFlowStatus.cs | 17 + .../DependencyFlowConfiguration.cs | 33 ++ .../IPullRequestActor.cs | 23 + .../ISubscriptionActor.cs | 22 + .../MergePolicyEvaluator.cs | 81 +++ .../NonBatchedPullRequestActor.cs | 21 + ...tConstructionService.DependencyFlow.csproj | 6 - .../Properties/AssemblyInfo.cs | 7 + .../PullRequestActor.cs | 91 ++++ .../PullRequestActorId.cs | 97 ++++ .../PullRequestBuilder.cs | 508 ++++++++++++++++++ .../PullRequestPolicyFailureNotifier.cs | 126 +++++ .../PullRequestStatus.cs | 13 + .../SubscriptionActor.cs | 162 ++++++ .../SubscriptionException.cs | 22 + .../SubscriptionPullRequestUpdate.cs | 17 + .../SubscriptionType.cs | 10 + .../BuildCoherencyInfoProcessor.cs | 88 +++ .../PullRequestCheckProcessor.cs | 26 + .../SubscriptionTriggerProcessor.cs | 123 +++++ .../SubscriptionUpdateProcessor.cs | 26 + .../WorkItems/ActorWorkItem.cs | 15 + .../WorkItems/InProgressPullRequest.cs | 36 ++ ...Item.cs => SubscriptionTriggerWorkItem.cs} | 4 +- .../WorkItems/SubscriptionUpdateWorkItem.cs | 37 ++ .../SubscriptionTriggerer.cs | 10 +- .../ReminderManager.cs | 8 +- .../ReminderManagerFactory.cs | 1 + .../WorkItemConfiguration.cs | 1 + .../SubscriptionsController20200220Tests.cs | 4 +- .../ActorTests.cs | 149 +++++ .../Disposable.cs | 24 + .../MockRedisCache.cs | 52 ++ .../MockRedisCacheFactory.cs | 21 + .../MockReminderManager.cs | 35 ++ .../MockReminderManagerFactory.cs | 17 + ...ructionService.DependencyFlow.Tests.csproj | 22 + .../PullRequestBuilderTests.cs | 316 +++++++++++ .../PullRequestPolicyFailureNotifierTests.cs | 280 ++++++++++ .../SubscriptionActorTests.cs | 107 ++++ .../SubscriptionOrPullRequestActorTests.cs | 166 ++++++ .../TestsWithMocks.cs | 30 ++ .../TestsWithServices.cs | 37 ++ .../VerifyableMockRepository.cs | 21 + .../SubscriptionTriggererTests.cs | 10 +- 51 files changed, 2990 insertions(+), 101 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeFlowStatus.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/MergePolicyEvaluator.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestBuilder.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestPolicyFailureNotifier.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestStatus.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionException.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionPullRequestUpdate.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionType.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/BuildCoherencyInfoProcessor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/{UpdateSubscriptionWorkItem.cs => SubscriptionTriggerWorkItem.cs} (82%) create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/Disposable.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/MockRedisCache.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/MockRedisCacheFactory.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/MockReminderManagerFactory.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/ProductConstructionService.DependencyFlow.Tests.csproj create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/TestsWithMocks.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/TestsWithServices.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/VerifyableMockRepository.cs diff --git a/arcade-services.sln b/arcade-services.sln index 7c6548ad3a..bf6cf67856 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -136,6 +136,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.DependencyFlow", "src\ProductConstructionService\ProductConstructionService.DependencyFlow\ProductConstructionService.DependencyFlow.csproj", "{E312686C-A134-486F-9F62-89CE6CA34702}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.DependencyFlow.Tests", "test\ProductConstructionService.DependencyFlow.Tests\ProductConstructionService.DependencyFlow.Tests.csproj", "{045BBB97-5A75-443A-8447-2035CBC0549A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -651,6 +653,18 @@ Global {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x64.Build.0 = Release|Any CPU {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x86.ActiveCfg = Release|Any CPU {E312686C-A134-486F-9F62-89CE6CA34702}.Release|x86.Build.0 = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|x64.ActiveCfg = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|x64.Build.0 = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|x86.ActiveCfg = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Debug|x86.Build.0 = Debug|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|Any CPU.Build.0 = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x64.ActiveCfg = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x64.Build.0 = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.ActiveCfg = Release|Any CPU + {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -702,6 +716,7 @@ Global {90C7747B-EBEF-4CF5-92A7-7856A3A13CAA} = {243A4561-BF35-405A-AF12-AC57BB27796D} {29A75658-2DC4-4E85-8A53-97198F00F28D} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} {E312686C-A134-486F-9F62-89CE6CA34702} = {243A4561-BF35-405A-AF12-AC57BB27796D} + {045BBB97-5A75-443A-8447-2035CBC0549A} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index ff806c758f..e89f6e89fa 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -187,7 +187,7 @@ where sub.Enabled if (subscriptionToUpdate != null) { - await _workItemProducerFactory.CreateProducer().ProduceWorkItemAsync(new() + await _workItemProducerFactory.CreateProducer().ProduceWorkItemAsync(new() { SubscriptionId = subscriptionToUpdate.Id, BuildId = buildId @@ -209,7 +209,7 @@ public virtual async Task TriggerDailyUpdateAsync() .ToListAsync()) .Where(s => (int)s.PolicyObject.UpdateFrequency == (int)UpdateFrequency.EveryDay); - var workitemProducer = _workItemProducerFactory.CreateProducer(); + var workitemProducer = _workItemProducerFactory.CreateProducer(); foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs index fffec48dd3..9343dca97c 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/BuildsController.cs @@ -338,77 +338,4 @@ await _workItemProducerFactory.CreateProducer() }, new Build(buildModel)); } - - // TODO PORT THIS TO A WORKITEM PROCESSOR: - /* - private class BuildCoherencyInfoWorkItem : IBackgroundWorkItem - { - private readonly BuildAssetRegistryContext _context; - private readonly IRemoteFactory _remoteFactory; - private readonly IBasicBarClient _barClient; - private readonly ILogger _logger; - - public BuildCoherencyInfoWorkItem( - BuildAssetRegistryContext context, - IRemoteFactory remoteFactory, - IBasicBarClient barClient, - ILogger logger) - { - _context = context; - _remoteFactory = remoteFactory; - _barClient = barClient; - _logger = logger; - } - - public async Task ProcessAsync(JToken argumentToken) - { - // This method is called asynchronously whenever a new build is inserted in BAR. - // It's goal is to compute the incoherent dependencies that the build have and - // persist the list of them in BAR. - - var buildId = argumentToken.Value(); - var graphBuildOptions = new DependencyGraphBuildOptions() - { - IncludeToolset = false, - LookupBuilds = false, - NodeDiff = NodeDiff.None - }; - - try - { - Maestro.Data.Models.Build build = await _context.Builds.FindAsync(buildId); - - DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( - _remoteFactory, - _barClient, - build.GitHubRepository ?? build.AzureDevOpsRepository, - build.Commit, - graphBuildOptions, - _logger); - - var incoherencies = new List(); - - foreach (var incoherence in graph.IncoherentDependencies) - { - incoherencies.Add(new Maestro.Data.Models.BuildIncoherence - { - Name = incoherence.Name, - Version = incoherence.Version, - Repository = incoherence.RepoUri, - Commit = incoherence.Commit - }); - } - - _context.Entry(build).Reload(); - build.Incoherencies = incoherencies; - - _context.Builds.Update(build); - await _context.SaveChangesAsync(); - } - catch (Exception e) - { - _logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {buildId}"); - } - } - }*/ } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index c5aab43e08..e0165bfe30 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -38,6 +38,7 @@ using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.DependencyFlow.WorkItemProcessors; using ProductConstructionService.WorkItems; +using ProductConstructionService.DependencyFlow; namespace ProductConstructionService.Api; @@ -75,7 +76,7 @@ static PcsStartup() { var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); - var workItemProducer = context.GetService().CreateProducer(); + var workItemProducer = context.GetService().CreateProducer(); var subscriptionIdGenerator = context.GetService(); BuildChannel entity = entry.Entity; @@ -164,12 +165,12 @@ internal static async Task ConfigurePcs( builder.AddBuildAssetRegistry(); builder.AddWorkItemQueues(azureCredential, waitForInitialization: initializeService); + builder.AddDependencyFlowProcessors(); builder.AddVmrRegistrations(gitHubToken); builder.AddMaestroApiClient(managedIdentityId); builder.AddGitHubClientFactory(); builder.Services.AddGitHubTokenProvider(); builder.Services.AddScoped(); - builder.Services.AddWorkItemProcessor(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.Configure(_ => { }); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs new file mode 100644 index 0000000000..5bb9581d9e --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.DependencyInjection; + +namespace ProductConstructionService.DependencyFlow; + +public interface IActorFactory +{ + IPullRequestActor CreatePullRequestActor(PullRequestActorId actorId); + + ISubscriptionActor CreateSubscriptionActor(Guid subscriptionId); +} + +internal class ActorFactory : IActorFactory +{ + private readonly IServiceProvider _serviceProvider; + + public ActorFactory(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public IPullRequestActor CreatePullRequestActor(PullRequestActorId actorId) => actorId switch + { + BatchedPullRequestActorId batched => ActivatorUtilities.CreateInstance(_serviceProvider, batched), + NonBatchedPullRequestActorId nonBatched => ActivatorUtilities.CreateInstance(_serviceProvider, nonBatched), + _ => throw new NotImplementedException() + }; + + public ISubscriptionActor CreateSubscriptionActor(Guid subscriptionId) + { + return ActivatorUtilities.CreateInstance(_serviceProvider, subscriptionId); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs new file mode 100644 index 0000000000..e611b0c130 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Maestro.Data.Models; + +namespace ProductConstructionService.DependencyFlow; + +/// +/// A for batched subscriptions that reads its Target and Merge Policies +/// from the configuration for a repository +/// +internal class BatchedPullRequestActor : PullRequestActor +{ + private readonly BatchedPullRequestActorId _id; + private readonly BuildAssetRegistryContext _context; + + public BatchedPullRequestActor( + BatchedPullRequestActorId id, + BuildAssetRegistryContext context) + : base(id) + { + _id = id; + _context = context; + } + + protected override Task<(string repository, string branch)> GetTargetAsync() + { + return Task.FromResult((_id.Repository, _id.Branch)); + } + + protected override async Task> GetMergePolicyDefinitions() + { + RepositoryBranch? repositoryBranch = await _context.RepositoryBranches.FindAsync(_id.Repository, _id.Branch); + return repositoryBranch?.PolicyObject?.MergePolicies ?? []; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeFlowStatus.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeFlowStatus.cs new file mode 100644 index 0000000000..02331bab35 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeFlowStatus.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; + +#nullable disable +namespace ProductConstructionService.DependencyFlow; + +[DataContract] +public class CodeFlowStatus +{ + [DataMember] + public string PrBranch { get; set; } + + [DataMember] + public string SourceSha { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs new file mode 100644 index 0000000000..f723b6cfa6 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.DataProviders; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using ProductConstructionService.DependencyFlow.WorkItemProcessors; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow; + +public static class DependencyFlowConfiguration +{ + public static void AddDependencyFlowProcessors(this IHostApplicationBuilder builder) + => builder.Services.AddDependencyFlowProcessors(); + + public static void AddDependencyFlowProcessors(this IServiceCollection services) + { + services.TryAddTransient(); + services.TryAddSingleton(); + services.TryAddTransient(); + services.TryAddScoped(); + + services.AddWorkItemProcessor(); + services.AddWorkItemProcessor(); + services.AddWorkItemProcessor(); + services.AddWorkItemProcessor(); + services.AddWorkItemProcessor(); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs new file mode 100644 index 0000000000..70a1aabafa --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.DependencyFlow.WorkItems; + +namespace ProductConstructionService.DependencyFlow; + +public interface IPullRequestActor +{ + Task<(InProgressPullRequest? pr, bool canUpdate)> SynchronizeInProgressPullRequestAsync( + InProgressPullRequest pullRequestCheck); + + Task ProcessPendingUpdatesAsync( + SubscriptionUpdateWorkItem update); + + Task UpdateAssetsAsync( + Guid subscriptionId, + SubscriptionType type, + int buildId, + string sourceRepo, + string sourceSha, + List assets); +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs new file mode 100644 index 0000000000..8969b35ab6 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Contracts; +using Microsoft.ServiceFabric.Actors; + +namespace ProductConstructionService.DependencyFlow; + +public interface ISubscriptionActor : IActor +{ + Task UpdateSubscriptionAsync(int buildId); + + Task UpdateForMergedPullRequestAsync(int updateBuildId); + + Task AddDependencyFlowEventAsync( + int updateBuildId, + DependencyFlowEventType flowEvent, + DependencyFlowEventReason reason, + MergePolicyCheckResult policy, + string flowType, + string? url); +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/MergePolicyEvaluator.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/MergePolicyEvaluator.cs new file mode 100644 index 0000000000..82787ee184 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/MergePolicyEvaluator.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Maestro.Contracts; +using Maestro.Data.Models; +using Maestro.MergePolicies; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Logging; +using Microsoft.Extensions.Logging; + +namespace ProductConstructionService.DependencyFlow; + +internal interface IMergePolicyEvaluator +{ + Task EvaluateAsync( + IPullRequest pr, + IRemote darc, + IReadOnlyList policyDefinitions); +} + +internal class MergePolicyEvaluator : IMergePolicyEvaluator +{ + private readonly OperationManager _operations; + + public MergePolicyEvaluator(IEnumerable mergePolicies, OperationManager operations, ILogger logger) + { + MergePolicyBuilders = mergePolicies.ToImmutableDictionary(p => p.Name); + Logger = logger; + _operations = operations; + } + + public IImmutableDictionary MergePolicyBuilders { get; } + public ILogger Logger { get; } + + public async Task EvaluateAsync( + IPullRequest pr, + IRemote darc, + IReadOnlyList policyDefinitions) + { + var results = new List(); + foreach (MergePolicyDefinition definition in policyDefinitions) + { + if (MergePolicyBuilders.TryGetValue(definition.Name, out IMergePolicyBuilder? policyBuilder)) + { + using var oDef = _operations.BeginOperation("Evaluating Merge Definition {policyName}", definition.Name); + var policies = await policyBuilder.BuildMergePoliciesAsync(new MergePolicyProperties(definition.Properties), pr); + foreach (var policy in policies) + { + using var oPol = _operations.BeginOperation("Evaluating Merge Policy {policyName}", policy.Name); + results.Add(await policy.EvaluateAsync(pr, darc)); + } + } + else + { + var notImplemented = new NotImplementedMergePolicy(definition.Name); + results.Add(new MergePolicyEvaluationResult(MergePolicyEvaluationStatus.Failure, $"Unknown Merge Policy: '{definition.Name}'", string.Empty, notImplemented)); + } + } + + return new MergePolicyEvaluationResults(results); + } + + private class NotImplementedMergePolicy : MergePolicy + { + private readonly string _definitionName; + + public NotImplementedMergePolicy(string definitionName) + { + _definitionName = definitionName; + } + + public override string DisplayName => $"Not implemented merge policy '{_definitionName}'"; + + public override Task EvaluateAsync(IPullRequest pr, IRemote darc) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs new file mode 100644 index 0000000000..6099f784a0 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; + +namespace ProductConstructionService.DependencyFlow; + +/// +/// A that reads its Merge Policies and Target information from a +/// non-batched subscription object +/// +internal class NonBatchedPullRequestActor : PullRequestActor +{ + public NonBatchedPullRequestActor(NonBatchedPullRequestActorId id) + : base(id) + { + } + + protected override Task> GetMergePolicyDefinitions() => throw new NotImplementedException(); + protected override Task<(string repository, string branch)> GetTargetAsync() => throw new NotImplementedException(); +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj index 1a98faf9dc..5331d100c9 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj @@ -8,12 +8,6 @@ False - - - - - - diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..3eed7b6adb --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("ProductConstructionService.DependencyFlow.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs new file mode 100644 index 0000000000..33d34acf8c --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs @@ -0,0 +1,91 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; +using ProductConstructionService.DependencyFlow.WorkItems; + +using Asset = Maestro.Contracts.Asset; + +namespace ProductConstructionService.DependencyFlow; + +/// +/// A service fabric actor implementation that is responsible for creating and updating pull requests for dependency +/// updates. +/// +internal abstract class PullRequestActor : IPullRequestActor +{ + private readonly PullRequestActorId _id; + + /// + /// Creates a new PullRequestActor + /// + public PullRequestActor( + PullRequestActorId id) + { + _id = id; + } + + protected abstract Task<(string repository, string branch)> GetTargetAsync(); + + protected abstract Task> GetMergePolicyDefinitions(); + + /// + /// Process any pending pull request updates. + /// + /// + /// True if updates have been applied; otherwise. + /// + public Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem update) + { + throw new NotImplementedException(); + } + + protected virtual Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgressPullRequest pr) + { + // Only do actual stuff in the non-batched implementation + return Task.CompletedTask; + } + + /// + /// Synchronizes an in progress pull request. + /// This will update current state if the pull request has been manually closed or merged. + /// This will evaluate merge policies on an in progress pull request and merge the pull request if policies allow. + /// + /// + /// A containing: + /// The current open pull request if one exists, and + /// if the open pull request can be updated; otherwise. + /// + public virtual Task<(InProgressPullRequest? pr, bool canUpdate)> SynchronizeInProgressPullRequestAsync(InProgressPullRequest pullRequestCheck) + { + throw new NotImplementedException(); + } + + /// + /// Applies or queues asset updates for the target repository and branch from the given build and list of assets. + /// + /// The id of the subscription the update comes from + /// The build that the updated assets came from + /// The commit hash that built the assets + /// The list of assets + /// + /// This function will queue updates if there is a pull request and it is currently not-updateable. + /// A pull request is considered "not-updateable" based on merge policies. + /// If at least one merge policy calls and + /// no merge policy calls then the pull request is considered + /// not-updateable. + /// + /// PRs are marked as non-updateable so that we can allow pull request checks to complete on a PR prior + /// to pushing additional commits. + /// + public Task UpdateAssetsAsync( + Guid subscriptionId, + SubscriptionType type, + int buildId, + string sourceRepo, + string sourceSha, + List assets) + { + throw new NotImplementedException(); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs new file mode 100644 index 0000000000..1e28bd8de1 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; + +namespace ProductConstructionService.DependencyFlow; + +public class ActorId +{ + public string Id { get; } + + public ActorId(string id) + { + Id = id; + } + + public override string ToString() => Id.ToString(); + + public override bool Equals(object? obj) => Id.Equals((obj as ActorId)?.Id); + + public override int GetHashCode() => Id.GetHashCode(); +} + +public class NonBatchedPullRequestActorId : PullRequestActorId +{ + public Guid SubscriptionId { get; } + + public NonBatchedPullRequestActorId(Guid subscriptionId) + : base(subscriptionId.ToString()) + { + SubscriptionId = subscriptionId; + } +} + +public class BatchedPullRequestActorId : PullRequestActorId +{ + public string Repository { get; } + public string Branch { get; } + + /// + /// Creates an identifying the PullRequestActor responsible for pull requests for all batched + /// subscriptions + /// targeting the (, ) pair. + /// + public BatchedPullRequestActorId(string repository, string branch) + : base(Encode(repository) + ":" + Encode(branch)) + { + Repository = repository; + Branch = branch; + } +} + +public abstract class PullRequestActorId : ActorId +{ + protected PullRequestActorId(string id) + : base(id) + { + } + + /// + /// Parses an created by into the (repository, branch) + /// pair that created it. + /// + public static PullRequestActorId Parse(string id) + { + if (Guid.TryParse(id, out var guid)) + { + return new NonBatchedPullRequestActorId(guid); + } + + var colonIndex = id.IndexOf(":", StringComparison.Ordinal); + if (colonIndex == -1) + { + throw new ArgumentException("Actor id not in correct format", nameof(id)); + } + + var repository = Decode(id.Substring(0, colonIndex)); + var branch = Decode(id.Substring(colonIndex + 1)); + return new BatchedPullRequestActorId(repository, branch); + } + + protected static string Encode(string repository) + { + return Convert.ToBase64String(Encoding.UTF8.GetBytes(repository)); + } + + protected static string Decode(string value) + { + return Encoding.UTF8.GetString(Convert.FromBase64String(value)); + } +} + +public enum ActorIdKind +{ + Guid, + String +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestBuilder.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestBuilder.cs new file mode 100644 index 0000000000..4737c9c3c6 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestBuilder.cs @@ -0,0 +1,508 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using System.Text.RegularExpressions; +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.Logging; +using ProductConstructionService.DependencyFlow.WorkItems; + +namespace ProductConstructionService.DependencyFlow; + +internal interface IPullRequestBuilder +{ + /// + /// Commit a dependency update to a target branch and calculate the PR description + /// + /// Version updates to apply + /// + /// A string writer that the PR description should be written to. If this an update + /// to an existing PR, this will contain the existing PR description. + /// + /// Remote factory for generating remotes based on repo uri + /// Target repository that the updates should be applied to + /// Target branch the updates should be to + Task CalculatePRDescriptionAndCommitUpdatesAsync( + List<(SubscriptionUpdateWorkItem update, List deps)> requiredUpdates, + string? currentDescription, + string targetRepository, + string newBranchName); + + /// + /// Compute the title for a pull request. + /// + /// Current in progress pull request information + /// Pull request title + Task GeneratePRTitleAsync( + InProgressPullRequest inProgressPr, + string targetBranch); + + /// + /// Generate the title for a code flow PR. + /// + Task GenerateCodeFlowPRTitleAsync( + SubscriptionUpdateWorkItem update, + string targetBranch); + + /// + /// Generate the description for a code flow PR. + /// + Task GenerateCodeFlowPRDescriptionAsync( + SubscriptionUpdateWorkItem update); +} + +internal class PullRequestBuilder : IPullRequestBuilder +{ + public const int GitHubComparisonShaLength = 10; + + // PR description markers + private const string DependencyUpdateBegin = "[DependencyUpdate]: <> (Begin)"; + private const string DependencyUpdateEnd = "[DependencyUpdate]: <> (End)"; + + private readonly BuildAssetRegistryContext _context; + private readonly IRemoteFactory _remoteFactory; + private readonly IBasicBarClient _barClient; + private readonly ILogger _logger; + + public PullRequestBuilder( + BuildAssetRegistryContext context, + IRemoteFactory remoteFactory, + IBasicBarClient barClient, + ILogger logger) + { + _context = context; + _remoteFactory = remoteFactory; + _barClient = barClient; + _logger = logger; + } + + public async Task GeneratePRTitleAsync(InProgressPullRequest inProgressPr, string targetBranch) + { + // Get the unique subscription IDs. It may be possible for a coherency update + // to not have any contained subscription. In this case + // we return a different title. + var uniqueSubscriptionIds = inProgressPr.ContainedSubscriptions + .Select(subscription => subscription.SubscriptionId) + .Distinct() + .ToArray(); + + if (uniqueSubscriptionIds.Length == 0) + { + return $"[{targetBranch}] Update dependencies to ensure coherency"; + } + + return await CreateTitleWithRepositories($"[{targetBranch}] Update dependencies from", uniqueSubscriptionIds); + } + + public async Task CalculatePRDescriptionAndCommitUpdatesAsync( + List<(SubscriptionUpdateWorkItem update, List deps)> requiredUpdates, + string? currentDescription, + string targetRepository, + string newBranchName) + { + StringBuilder description = new StringBuilder(currentDescription ?? "This pull request updates the following dependencies") + .AppendLine() + .AppendLine(); + var startingReferenceId = GetStartingReferenceId(description.ToString()); + + // First run through non-coherency and then do a coherency + // message if one exists. + var nonCoherencyUpdates = + requiredUpdates.Where(u => !u.update.IsCoherencyUpdate).ToList(); + // Should max one coherency update + (SubscriptionUpdateWorkItem update, List deps) coherencyUpdate = + requiredUpdates.Where(u => u.update.IsCoherencyUpdate).SingleOrDefault(); + + IRemote remote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + var locationResolver = new AssetLocationResolver(_barClient); + + // To keep a PR to as few commits as possible, if the number of + // non-coherency updates is 1 then combine coherency updates with those. + // Otherwise, put all coherency updates in a separate commit. + var combineCoherencyWithNonCoherency = nonCoherencyUpdates.Count == 1; + + foreach ((SubscriptionUpdateWorkItem update, List deps) in nonCoherencyUpdates) + { + var message = new StringBuilder(); + List dependenciesToCommit = deps; + await CalculateCommitMessage(update, deps, message); + var build = await _barClient.GetBuildAsync(update.BuildId) + ?? throw new Exception($"Failed to find build {update.BuildId} for subscription {update.SubscriptionId}"); + + if (combineCoherencyWithNonCoherency && coherencyUpdate.update != null) + { + await CalculateCommitMessage(coherencyUpdate.update, coherencyUpdate.deps, message); + AppendCoherencyUpdateDescription(description, coherencyUpdate.deps); + dependenciesToCommit.AddRange(coherencyUpdate.deps); + } + + var itemsToUpdate = dependenciesToCommit + .Select(du => du.To) + .ToList(); + + await locationResolver.AddAssetLocationToDependenciesAsync(itemsToUpdate); + + List committedFiles = await remote.CommitUpdatesAsync( + targetRepository, + newBranchName, + _remoteFactory, + _barClient, + itemsToUpdate, + message.ToString()); + + AppendBuildDescription(description, ref startingReferenceId, update, deps, committedFiles, build); + } + + // If the coherency update wasn't combined, then + // add it now + if (!combineCoherencyWithNonCoherency && coherencyUpdate.update != null) + { + var message = new StringBuilder(); + var build = await _barClient.GetBuildAsync(coherencyUpdate.update.BuildId); + await CalculateCommitMessage(coherencyUpdate.update, coherencyUpdate.deps, message); + AppendCoherencyUpdateDescription(description, coherencyUpdate.deps); + + var itemsToUpdate = coherencyUpdate.deps + .Select(du => du.To) + .ToList(); + + await locationResolver.AddAssetLocationToDependenciesAsync(itemsToUpdate); + await remote.CommitUpdatesAsync( + targetRepository, + newBranchName, + _remoteFactory, + _barClient, + itemsToUpdate, + message.ToString()); + } + + // If the coherency algorithm failed and there are no non-coherency updates and + // we create an empty commit that describes an issue. + if (requiredUpdates.Count == 0) + { + var message = "Failed to perform coherency update for one or more dependencies."; + await remote.CommitUpdatesAsync(targetRepository, newBranchName, _remoteFactory, _barClient, [], message); + return $"Coherency update: {message} Please review the GitHub checks or run `darc update-dependencies --coherency-only` locally against {newBranchName} for more information."; + } + + return description.ToString(); + } + + public async Task GenerateCodeFlowPRTitleAsync( + SubscriptionUpdateWorkItem update, + string targetBranch) + { + return await CreateTitleWithRepositories($"[{targetBranch}] Source code changes from ", [update.SubscriptionId]); + } + + public async Task GenerateCodeFlowPRDescriptionAsync(SubscriptionUpdateWorkItem update) + { + var build = await _barClient.GetBuildAsync(update.BuildId) + ?? throw new Exception($"Failed to find build {update.BuildId} for subscription {update.SubscriptionId}"); + + return + $""" + {GetStartMarker(update.SubscriptionId)} + + This pull request is bringing source changes from **{update.SourceRepo}**. + + - **Subscription**: {update.SubscriptionId} + - **Build**: {build.AzureDevOpsBuildNumber} + - **Date Produced**: {build.DateProduced.ToUniversalTime():MMMM d, yyyy h:mm:ss tt UTC} + - **Commit**: {build.Commit} + - **Branch**: {build.GetBranch()} + + {GetEndMarker(update.SubscriptionId)} + """; + } + + /// + /// Append build description to the PR description + /// + /// Description to extend + /// Counter for references + /// Update + /// Dependencies updated + /// List of commited files + /// Build + /// + /// Because PRs tend to be live for short periods of time, we can put more information + /// in the description than the commit message without worrying that links will go stale. + /// + private void AppendBuildDescription(StringBuilder description, ref int startingReferenceId, SubscriptionUpdateWorkItem update, List deps, List? committedFiles, Microsoft.DotNet.Maestro.Client.Models.Build build) + { + var changesLinks = new List(); + + var sourceRepository = update.SourceRepo; + Guid updateSubscriptionId = update.SubscriptionId; + var sectionStartMarker = GetStartMarker(updateSubscriptionId); + var sectionEndMarker = GetEndMarker(updateSubscriptionId); + var sectionStartIndex = RemovePRDescriptionSection(description, sectionStartMarker, sectionEndMarker); + + var subscriptionSection = new StringBuilder() + .AppendLine(sectionStartMarker) + .AppendLine($"## From {sourceRepository}") + .AppendLine($"- **Subscription**: {updateSubscriptionId}") + .AppendLine($"- **Build**: {build.AzureDevOpsBuildNumber}") + .AppendLine($"- **Date Produced**: {build.DateProduced.ToUniversalTime():MMMM d, yyyy h:mm:ss tt UTC}") + // This is duplicated from the files changed, but is easier to read here. + .AppendLine($"- **Commit**: {build.Commit}"); + + var branch = build.AzureDevOpsBranch ?? build.GitHubBranch; + if (!string.IsNullOrEmpty(branch)) + { + subscriptionSection.AppendLine($"- **Branch**: {branch}"); + } + + subscriptionSection + .AppendLine() + .AppendLine(DependencyUpdateBegin) + .AppendLine() + .AppendLine($"- **Updates**:"); + + var shaRangeToLinkId = new Dictionary<(string from, string to), int>(); + + foreach (DependencyUpdate dep in deps) + { + if (!shaRangeToLinkId.ContainsKey((dep.From.Commit, dep.To.Commit))) + { + var changesUri = string.Empty; + try + { + changesUri = GetChangesURI(dep.To.RepoUri, dep.From.Commit, dep.To.Commit); + } + catch (ArgumentNullException e) + { + _logger.LogError(e, $"Failed to create SHA comparison link for dependency {dep.To.Name} during asset update for subscription {update.SubscriptionId}"); + } + shaRangeToLinkId.Add((dep.From.Commit, dep.To.Commit), startingReferenceId + changesLinks.Count); + changesLinks.Add(changesUri); + } + subscriptionSection.AppendLine($" - **{dep.To.Name}**: [from {dep.From.Version} to {dep.To.Version}][{shaRangeToLinkId[(dep.From.Commit, dep.To.Commit)]}]"); + } + + subscriptionSection.AppendLine(); + for (var i = 0; i < changesLinks.Count; i++) + { + subscriptionSection.AppendLine($"[{i + startingReferenceId}]: {changesLinks[i]}"); + } + + subscriptionSection + .AppendLine() + .AppendLine(DependencyUpdateEnd) + .AppendLine(); + + UpdatePRDescriptionDueConfigFiles(committedFiles, subscriptionSection); + + subscriptionSection + .AppendLine() + .AppendLine(sectionEndMarker); + + description.Insert(sectionStartIndex, subscriptionSection.ToString()); + description.AppendLine(); + + startingReferenceId += changesLinks.Count; + } + + /// + /// Append coherency update description to the PR description + /// + /// Description to extend + /// Dependencies updated + /// + /// Because PRs tend to be live for short periods of time, we can put more information + /// in the description than the commit message without worrying that links will go stale. + /// + private static void AppendCoherencyUpdateDescription(StringBuilder description, List dependencies) + { + var sectionStartMarker = "[marker]: <> (Begin:Coherency Updates)"; + var sectionEndMarker = "[marker]: <> (End:Coherency Updates)"; + var sectionStartIndex = RemovePRDescriptionSection(description, sectionStartMarker, sectionEndMarker); + + var coherencySection = new StringBuilder() + .AppendLine(sectionStartMarker) + .AppendLine("## Coherency Updates") + .AppendLine() + .AppendLine("The following updates ensure that dependencies with a *CoherentParentDependency*") + .AppendLine("attribute were produced in a build used as input to the parent dependency's build.") + .AppendLine("See [Dependency Description Format](https://github.com/dotnet/arcade/blob/master/Documentation/DependencyDescriptionFormat.md#dependency-description-overview)") + .AppendLine() + .AppendLine(DependencyUpdateBegin) + .AppendLine() + .AppendLine("- **Coherency Updates**:"); + + foreach (DependencyUpdate dep in dependencies) + { + coherencySection.AppendLine($" - **{dep.To.Name}**: from {dep.From.Version} to {dep.To.Version} (parent: {dep.To.CoherentParentDependencyName})"); + } + + coherencySection + .AppendLine() + .AppendLine(DependencyUpdateEnd) + .AppendLine() + .AppendLine(sectionEndMarker); + + description.Insert(sectionStartIndex, coherencySection.ToString()); + description.AppendLine(); + } + + private async Task CalculateCommitMessage(SubscriptionUpdateWorkItem update, List deps, StringBuilder message) + { + if (update.IsCoherencyUpdate) + { + message.AppendLine("Dependency coherency updates"); + message.AppendLine(); + message.AppendLine(string.Join(",", deps.Select(p => p.To.Name))); + message.AppendLine($" From Version {deps[0].From.Version} -> To Version {deps[0].To.Version} (parent: {deps[0].To.CoherentParentDependencyName}"); + } + else + { + var sourceRepository = update.SourceRepo; + var build = await _barClient.GetBuildAsync(update.BuildId); + message.AppendLine($"Update dependencies from {sourceRepository} build {build?.AzureDevOpsBuildNumber}"); + message.AppendLine(); + message.AppendLine(string.Join(" , ", deps.Select(p => p.To.Name))); + message.AppendLine($" From Version {deps[0].From.Version} -> To Version {deps[0].To.Version}"); + } + + message.AppendLine(); + } + + private static void UpdatePRDescriptionDueConfigFiles(List? committedFiles, StringBuilder globalJsonSection) + { + GitFile? globalJsonFile = committedFiles?. + Where(gf => gf.FilePath.Equals("global.json", StringComparison.OrdinalIgnoreCase)). + FirstOrDefault(); + + // The list of committedFiles can contain the `global.json` file (and others) + // even though no actual change was made to the file and therefore there is no + // metadata for it. + if (globalJsonFile?.Metadata != null) + { + var hasSdkVersionUpdate = globalJsonFile.Metadata.ContainsKey(GitFileMetadataName.SdkVersionUpdate); + var hasToolsDotnetUpdate = globalJsonFile.Metadata.ContainsKey(GitFileMetadataName.ToolsDotNetUpdate); + + globalJsonSection.AppendLine("- **Updates to .NET SDKs:**"); + + if (hasSdkVersionUpdate) + { + globalJsonSection.AppendLine($" - Updates sdk.version to " + + $"{globalJsonFile.Metadata[GitFileMetadataName.SdkVersionUpdate]}"); + } + + if (hasToolsDotnetUpdate) + { + globalJsonSection.AppendLine($" - Updates tools.dotnet to " + + $"{globalJsonFile.Metadata[GitFileMetadataName.ToolsDotNetUpdate]}"); + } + } + } + + private static int RemovePRDescriptionSection(StringBuilder description, string sectionStartMarker, string sectionEndMarker) + { + var descriptionString = description.ToString(); + var sectionStartIndex = descriptionString.IndexOf(sectionStartMarker); + var sectionEndIndex = descriptionString.IndexOf(sectionEndMarker); + + if (sectionStartIndex != -1 && sectionEndIndex != -1) + { + sectionEndIndex += sectionEndMarker.Length; + description.Remove(sectionStartIndex, sectionEndIndex - sectionStartIndex); + return sectionStartIndex; + } + + // if either marker is missing, just append at end and don't remove anything + // from the description + return description.Length; + } + + /// + /// Goes through the description and finds the biggest reference id. This is needed when updating an exsiting PR. + /// + public static int GetStartingReferenceId(string description) + { + //The regex is matching numbers surrounded by square brackets that have a colon and something after it. + //The regex captures these numbers + //example: given [23]:sometext as input, it will attempt to capture "23" + var regex = new Regex("(?<=^\\[)\\d+(?=\\]:.+)", RegexOptions.Multiline); + + return regex.Matches(description.ToString()) + .Select(m => int.Parse(m.ToString())) + .DefaultIfEmpty(0) + .Max() + 1; + } + + public static string GetChangesURI(string repoURI, string fromSha, string toSha) + { + ArgumentNullException.ThrowIfNull(repoURI); + ArgumentNullException.ThrowIfNull(fromSha); + ArgumentNullException.ThrowIfNull(toSha); + + if (repoURI.Contains("github.com")) + { + var fromShortSha = fromSha.Length > GitHubComparisonShaLength ? fromSha.Substring(0, GitHubComparisonShaLength) : fromSha; + var toShortSha = toSha.Length > GitHubComparisonShaLength ? toSha.Substring(0, GitHubComparisonShaLength) : toSha; + + return $"{repoURI}/compare/{fromShortSha}...{toShortSha}"; + } + + // Azdo commit comparison doesn't work with short shas + return $"{repoURI}/branches?baseVersion=GC{fromSha}&targetVersion=GC{toSha}&_a=files"; + } + + private async Task GetSourceRepositoryAsync(Guid subscriptionId) + { + Subscription? subscription = await _context.Subscriptions.FindAsync(subscriptionId); + return subscription?.SourceRepository; + } + + /// + /// Either inserts a full list of the repos involved (in a shortened form) + /// or just the number of repos that are involved if title is too long. + /// + /// Start of the title to append the list to + private async Task CreateTitleWithRepositories(string baseTitle, Guid[] subscriptionIds) + { + // Github title limit - 348 + // Azdo title limit - 419 + // maxTitleLength = 150 to fit 2/3 repo names in the title + const int maxTitleLength = 150; + var maxRepoListLength = maxTitleLength - baseTitle.Length; + const string delimiter = ", "; + + var repoNames = new List(); + var titleLength = 0; + foreach (Guid subscriptionId in subscriptionIds) + { + var repoName = await GetSourceRepositoryAsync(subscriptionId); + if (repoName == null) + { + continue; + } + + // Strip down repo name. + repoName = repoName + .Replace("https://github.com/", null) + .Replace("https://dev.azure.com/", null) + .Replace("_git/", null); + + repoNames.Add(repoName); + + titleLength += repoName.Length + delimiter.Length; + if (titleLength > maxRepoListLength) + { + return $"{baseTitle} {subscriptionIds.Length} repositories"; + } + } + + return $"{baseTitle} {string.Join(delimiter, repoNames.OrderBy(s => s))}"; + } + + private static string GetStartMarker(Guid subscriptionId) + => $"[marker]: <> (Begin:{subscriptionId})"; + + private static string GetEndMarker(Guid subscriptionId) + => $"[marker]: <> (End:{subscriptionId})"; +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestPolicyFailureNotifier.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestPolicyFailureNotifier.cs new file mode 100644 index 0000000000..66bff62874 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestPolicyFailureNotifier.cs @@ -0,0 +1,126 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.Extensions.Logging; +using ProductConstructionService.DependencyFlow.WorkItems; + +namespace ProductConstructionService.DependencyFlow; + +internal interface IPullRequestPolicyFailureNotifier +{ + Task TagSourceRepositoryGitHubContactsAsync(InProgressPullRequest pr); +} + +internal class PullRequestPolicyFailureNotifier : IPullRequestPolicyFailureNotifier +{ + private readonly ILogger _logger; + private readonly IRemoteFactory _remoteFactory; + private readonly IBasicBarClient _barClient; + private readonly IGitHubTokenProvider _gitHubTokenProvider; + private readonly IGitHubClientFactory _gitHubClientFactory; + + public PullRequestPolicyFailureNotifier( + IGitHubTokenProvider gitHubTokenProvider, + IGitHubClientFactory gitHubClientFactory, + IRemoteFactory darcFactory, + IBasicBarClient barClient, + ILogger logger) + { + _logger = logger; + _gitHubTokenProvider = gitHubTokenProvider; + _gitHubClientFactory = gitHubClientFactory; + _remoteFactory = darcFactory; + _barClient = barClient; + } + + public async Task TagSourceRepositoryGitHubContactsAsync(InProgressPullRequest pr) + { + // We'll try to notify the source repo if the subscription provided a list of aliases to tag. + // The API checks when creating / updating subscriptions that any resolve-able logins are in the + // "Microsoft" Github org, so we can safely use them in any comment. + if (pr.SourceRepoNotified == true) + { + _logger.LogInformation("Skipped notifying source repository for {url}'s failed policies, as it has already been tagged", pr.Url); + return; + } + + var subscriptionFromPr = pr.ContainedSubscriptions.FirstOrDefault(); + if (subscriptionFromPr == null) + { + _logger.LogWarning("Unable to get any contained subscriptions from this PR for notification; skipping attempts to notify."); + pr.SourceRepoNotified = true; + return; + } + + // In practice these all contain the same subscription id, the property is more like "containedBuildsAndTheirSubscriptions" + _logger.LogInformation("PR contains {count} builds. Using first ({subscription}) for notification tagging.", + pr.ContainedSubscriptions.Count, + subscriptionFromPr.SubscriptionId); + + (var owner, var repo, var prIssueId) = GitHubClient.ParsePullRequestUri(pr.Url); + if (owner == null || repo == null || prIssueId == 0) + { + _logger.LogInformation("Unable to parse pull request '{url}' (typically due to Azure DevOps pull requests), will not notify on this PR.", pr.Url); + pr.SourceRepoNotified = true; + return; + } + + var darcRemote = await _remoteFactory.GetRemoteAsync($"https://github.com/{owner}/{repo}", _logger); + var darcSubscriptionObject = await _barClient.GetSubscriptionAsync(subscriptionFromPr.SubscriptionId); + var sourceRepository = darcSubscriptionObject.SourceRepository; + var targetRepository = darcSubscriptionObject.TargetRepository; + + // If we're here, there are failing checks, but if the only checks that failed were Maestro Merge Policy checks, we'll skip informing until something else fails too. + var prChecks = await darcRemote.GetPullRequestChecksAsync(pr.Url); + var failedPrChecks = prChecks.Where(p => !p.IsMaestroMergePolicy && (p.Status == CheckState.Failure || p.Status == CheckState.Error)).AsEnumerable(); + if (!failedPrChecks.Any()) + { + _logger.LogInformation("All failing or error state checks are 'Maestro Merge Policy'-type checks, not notifying subscribed users."); + return; + } + + var tagsToNotify = new List(); + if (!string.IsNullOrEmpty(darcSubscriptionObject.PullRequestFailureNotificationTags)) + { + tagsToNotify.AddRange(darcSubscriptionObject.PullRequestFailureNotificationTags.Split(';', StringSplitOptions.RemoveEmptyEntries)); + } + + if (tagsToNotify.Count == 0) + { + _logger.LogInformation("Found no matching tags for source '{sourceRepo}' to target '{targetRepo}' on channel '{channel}'. ", sourceRepository, targetRepository, darcSubscriptionObject.Channel); + return; + } + + // At this point we definitely have notifications to make, so do it. + _logger.LogInformation("Found {count} matching tags for source '{sourceRepo}' to target '{targetRepo}' on channel '{channel}'. ", tagsToNotify.Count, sourceRepository, targetRepository, darcSubscriptionObject.Channel); + + // To ensure GitHub notifies the people / teams on the list, forcibly check they are inserted with a preceding '@' + for (var i = 0; i < tagsToNotify.Count; i++) + { + if (!tagsToNotify[i].StartsWith('@')) + { + tagsToNotify[i] = $"@{tagsToNotify[i]}"; + } + } + + var githubToken = await _gitHubTokenProvider.GetTokenForRepository(targetRepository); + var gitHubClient = _gitHubClientFactory.CreateGitHubClient(githubToken); + + var sourceRepoNotificationComment = @$""" + #### Notification for subscribed users from {sourceRepository}: + + {string.Join($", {Environment.NewLine}", tagsToNotify)} + + #### Action requested: Please take a look at this failing automated dependency-flow pull request's checks; failures may be related to changes which originated in your repo. + + - This pull request contains changes from your source repo ({sourceRepository}) and seems to have failed checks in this PR. Please take a peek at the failures and comment if they seem relevant to your changes. + - If you're being tagged in this comment it is due to an entry in the related Maestro Subscription of the Build Asset Registry. If you feel this entry has added your GitHub login or your GitHub team in error, please update the subscription to reflect this. + - For more details, please read [the Arcade Darc documentation](https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#update-subscription) + """; + + await gitHubClient.Issue.Comment.Create(owner, repo, prIssueId, sourceRepoNotificationComment); + pr.SourceRepoNotified = true; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestStatus.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestStatus.cs new file mode 100644 index 0000000000..869070e3f6 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestStatus.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.DependencyFlow; + +public enum PullRequestStatus +{ + Invalid = 0, + UnknownPR = 1, + Completed = 2, + InProgressCanUpdate = 3, + InProgressCannotUpdate = 4, +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs new file mode 100644 index 0000000000..342ffb0e8c --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs @@ -0,0 +1,162 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Contracts; +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Asset = Maestro.Contracts.Asset; + +namespace ProductConstructionService.DependencyFlow; + +internal class SubscriptionActor : ISubscriptionActor +{ + private readonly IActorFactory _actorFactory; + private readonly BuildAssetRegistryContext _context; + private readonly ILogger _logger; + private readonly Guid _subscriptionId; + + public SubscriptionActor( + BuildAssetRegistryContext context, + IActorFactory actorFactory, + ILogger logger, + Guid subscriptionId) + { + _context = context; + _actorFactory = actorFactory; + _logger = logger; + _subscriptionId = subscriptionId; + } + + public async Task UpdateForMergedPullRequestAsync(int updateBuildId) + { + _logger.LogInformation("Updating {subscriptionId} with latest build id {buildId}", _subscriptionId, updateBuildId); + Subscription? subscription = await _context.Subscriptions.FindAsync(_subscriptionId); + + if (subscription != null) + { + subscription.LastAppliedBuildId = updateBuildId; + _context.Subscriptions.Update(subscription); + await _context.SaveChangesAsync(); + return true; + } + else + { + _logger.LogInformation("Could not find subscription with ID {subscriptionId}. Skipping latestBuild update.", _subscriptionId); + return false; + } + } + + public async Task AddDependencyFlowEventAsync( + int updateBuildId, + DependencyFlowEventType flowEvent, + DependencyFlowEventReason reason, + MergePolicyCheckResult policy, + string flowType, + string? url) + { + var updateReason = reason == DependencyFlowEventReason.New || reason == DependencyFlowEventReason.AutomaticallyMerged + ? reason.ToString() + : $"{reason}{policy}"; + + _logger.LogInformation( + "Adding dependency flow event for {subscriptionId} with {flowEvent} {updateReason} {flowType}", + _subscriptionId, + flowEvent, + updateReason, + flowType); + + Subscription? subscription = await _context.Subscriptions.FindAsync(_subscriptionId); + if (subscription != null) + { + var dfe = new DependencyFlowEvent + { + SourceRepository = subscription.SourceRepository, + TargetRepository = subscription.TargetRepository, + ChannelId = subscription.ChannelId, + BuildId = updateBuildId, + Timestamp = DateTimeOffset.UtcNow, + Event = flowEvent.ToString(), + Reason = updateReason, + FlowType = flowType, + Url = url, + }; + _context.DependencyFlowEvents.Add(dfe); + await _context.SaveChangesAsync(); + return true; + } + else + { + _logger.LogInformation("Could not find subscription with ID {subscriptionId}. Skipping adding dependency flow event.", _subscriptionId); + return false; + } + } + + public async Task UpdateSubscriptionAsync(int buildId) + { + Subscription? subscription = await _context.Subscriptions.FindAsync(_subscriptionId); + + if (subscription == null) + { + _logger.LogWarning("Could not find subscription with ID {subscriptionId}. Skipping update.", _subscriptionId); + return; + } + + await AddDependencyFlowEventAsync( + buildId, + DependencyFlowEventType.Fired, + DependencyFlowEventReason.New, + MergePolicyCheckResult.PendingPolicies, + "PR", + null); + + _logger.LogInformation("Looking up build {buildId}", buildId); + + Build build = await _context.Builds.Include(b => b.Assets) + .ThenInclude(a => a.Locations) + .FirstAsync(b => b.Id == buildId); + + IPullRequestActor pullRequestActor; + + if (subscription.PolicyObject.Batchable) + { + _logger.LogInformation("Creating pull request actor for branch {branch} of {repository}", + subscription.TargetBranch, + subscription.TargetRepository); + + pullRequestActor = _actorFactory.CreatePullRequestActor( + new BatchedPullRequestActorId(subscription.TargetRepository, subscription.TargetBranch)); + } + else + { + _logger.LogInformation("Creating pull request actor for subscription {subscriptionId}", + _subscriptionId); + + pullRequestActor = _actorFactory.CreatePullRequestActor( + new NonBatchedPullRequestActorId(_subscriptionId)); + } + + var assets = build.Assets + .Select(a => new Asset + { + Name = a.Name, + Version = a.Version + }) + .ToList(); + + _logger.LogInformation("Running asset update for {subscriptionId}", _subscriptionId); + + await pullRequestActor.UpdateAssetsAsync( + _subscriptionId, + subscription.SourceEnabled + ? SubscriptionType.DependenciesAndSources + : SubscriptionType.Dependencies, + build.Id, + build.GitHubRepository ?? build.AzureDevOpsRepository, + build.Commit, + assets); + + _logger.LogInformation("Asset update complete for {subscriptionId}", _subscriptionId); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionException.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionException.cs new file mode 100644 index 0000000000..19678d600b --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionException.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.DependencyFlow; + +/// +/// Exception thrown when there is a failure updating a subscription that should be surfaced to users. +/// +internal class SubscriptionException : Exception +{ + public SubscriptionException(string message) : base(message) + { + } + + public SubscriptionException() : this("There was a problem updating the subscription") + { + } + + public SubscriptionException(string message, Exception innerException) : base(message, innerException) + { + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionPullRequestUpdate.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionPullRequestUpdate.cs new file mode 100644 index 0000000000..f7158f96a7 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionPullRequestUpdate.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; + +#nullable disable +namespace ProductConstructionService.DependencyFlow; + +[DataContract] +public class SubscriptionPullRequestUpdate +{ + [DataMember] + public Guid SubscriptionId { get; set; } + + [DataMember] + public int BuildId { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionType.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionType.cs new file mode 100644 index 0000000000..922b2e66da --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionType.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.DependencyFlow; + +public enum SubscriptionType +{ + Dependencies = 0, + DependenciesAndSources = 1, +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/BuildCoherencyInfoProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/BuildCoherencyInfoProcessor.cs new file mode 100644 index 0000000000..03d95a8ee3 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/BuildCoherencyInfoProcessor.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.Logging; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; + +public class BuildCoherencyInfoProcessor : WorkItemProcessor +{ + private readonly BuildAssetRegistryContext _context; + private readonly IRemoteFactory _remoteFactory; + private readonly IBasicBarClient _barClient; + private readonly ILogger _logger; + + public BuildCoherencyInfoProcessor( + BuildAssetRegistryContext context, + IRemoteFactory remoteFactory, + IBasicBarClient barClient, + ILogger logger) + { + _context = context; + _remoteFactory = remoteFactory; + _barClient = barClient; + _logger = logger; + } + + /// + /// This method is called asynchronously whenever a new build is inserted in BAR. + /// Its goal is to compute the incoherent dependencies that the build have + /// and persist the list of them in BAR. + /// + public override async Task ProcessWorkItemAsync( + BuildCoherencyInfoWorkItem workItem, + CancellationToken cancellationToken) + { + var graphBuildOptions = new DependencyGraphBuildOptions() + { + IncludeToolset = false, + LookupBuilds = false, + NodeDiff = NodeDiff.None + }; + + try + { + Maestro.Data.Models.Build? build = await _context.Builds.FindAsync([workItem.BuildId], cancellationToken); + + if (build == null) + { + _logger.LogError("Build {buildId} not found", workItem.BuildId); + return false; + } + + DependencyGraph graph = await DependencyGraph.BuildRemoteDependencyGraphAsync( + _remoteFactory, + _barClient, + build.GitHubRepository ?? build.AzureDevOpsRepository, + build.Commit, + graphBuildOptions, + _logger); + + var incoherencies = graph.IncoherentDependencies + .Select(incoherence => new Maestro.Data.Models.BuildIncoherence + { + Name = incoherence.Name, + Version = incoherence.Version, + Repository = incoherence.RepoUri, + Commit = incoherence.Commit + }) + .ToList(); + + _context.Entry(build).Reload(); + build.Incoherencies = incoherencies; + _context.Builds.Update(build); + await _context.SaveChangesAsync(cancellationToken); + } + catch (Exception e) + { + _logger.LogWarning(e, $"Problems computing the dependency incoherencies for BAR build {workItem.BuildId}"); + return false; + } + + return true; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs new file mode 100644 index 0000000000..644523b268 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; + +public class PullRequestCheckProcessor : WorkItemProcessor +{ + private readonly IActorFactory _actorFactory; + + public PullRequestCheckProcessor(IActorFactory actorFactory) + { + _actorFactory = actorFactory; + } + + public override async Task ProcessWorkItemAsync( + InProgressPullRequest workItem, + CancellationToken cancellationToken) + { + var actor = _actorFactory.CreatePullRequestActor(PullRequestActorId.Parse(workItem.ActorId)); + await actor.SynchronizeInProgressPullRequestAsync(workItem); + return true; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs new file mode 100644 index 0000000000..a548fd86f9 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs @@ -0,0 +1,123 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Microsoft.DotNet.Internal.Logging; +using Microsoft.Extensions.Logging; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; + +public class SubscriptionTriggerProcessor : WorkItemProcessor +{ + private readonly BuildAssetRegistryContext _context; + private readonly OperationManager _operations; + private readonly IActorFactory _actorFactory; + private readonly ILogger _logger; + + public SubscriptionTriggerProcessor( + BuildAssetRegistryContext context, + OperationManager operationManager, + IActorFactory actorFactory, + ILogger logger) + { + _context = context; + _operations = operationManager; + _actorFactory = actorFactory; + _logger = logger; + } + + public override async Task ProcessWorkItemAsync( + SubscriptionTriggerWorkItem workItem, + CancellationToken cancellationToken) + { + if (workItem.BuildId.HasValue && workItem.BuildId.Value > 0) + { + return await StartSubscriptionUpdateForSpecificBuildAsync( + workItem.SubscriptionId, + workItem.BuildId.Value); + } + + return await StartSubscriptionUpdateAsync(workItem.SubscriptionId); + } + + /// + /// Run a single subscription, only accept the build Id specified + /// + /// Subscription to run the update for. + /// BAR build id to run the update for + private async Task StartSubscriptionUpdateForSpecificBuildAsync(Guid subscriptionId, int buildId) + { + var subscriptionToUpdate = + (from sub in _context.Subscriptions + where sub.Id == subscriptionId + where sub.Enabled + let specificBuild = + sub.Channel.BuildChannels.Select(bc => bc.Build) + .Where(b => sub.SourceRepository == b.GitHubRepository || sub.SourceRepository == b.AzureDevOpsRepository) + .Where(b => b.Id == buildId) + .FirstOrDefault() + where specificBuild != null + select new + { + subscription = sub.Id, + specificBuild = specificBuild.Id + }).SingleOrDefault(); + + if (subscriptionToUpdate == null) + { + return true; + } + + return await UpdateSubscriptionAsync(subscriptionToUpdate.subscription, subscriptionToUpdate.specificBuild); + } + + /// + /// Run a single subscription, adopting the latest build's id + /// + /// Subscription to run the update for. + private async Task StartSubscriptionUpdateAsync(Guid subscriptionId) + { + var subscriptionToUpdate = + (from sub in _context.Subscriptions + where sub.Id == subscriptionId + where sub.Enabled + let latestBuild = + sub.Channel.BuildChannels.Select(bc => bc.Build) + .Where(b => (sub.SourceRepository == b.GitHubRepository || sub.SourceRepository == b.AzureDevOpsRepository)) + .OrderByDescending(b => b.DateProduced) + .FirstOrDefault() + where latestBuild != null + select new + { + subscription = sub.Id, + latestBuild = latestBuild.Id + }).SingleOrDefault(); + + if (subscriptionToUpdate == null) + { + return true; + } + + return await UpdateSubscriptionAsync(subscriptionToUpdate.subscription, subscriptionToUpdate.latestBuild); + } + + private async Task UpdateSubscriptionAsync(Guid subscriptionId, int buildId) + { + using (_operations.BeginOperation("Updating subscription '{subscriptionId}' with build '{buildId}'", subscriptionId, buildId)) + { + try + { + ISubscriptionActor actor = _actorFactory.CreateSubscriptionActor(subscriptionId); + await actor.UpdateSubscriptionAsync(buildId); + return true; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to update subscription '{subscriptionId}' with build '{buildId}'", subscriptionId, buildId); + return false; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs new file mode 100644 index 0000000000..ad2aaceb88 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; + +public class SubscriptionUpdateProcessor : WorkItemProcessor +{ + private readonly IActorFactory _actorFactory; + + public SubscriptionUpdateProcessor(IActorFactory actorFactory) + { + _actorFactory = actorFactory; + } + + public override async Task ProcessWorkItemAsync( + SubscriptionUpdateWorkItem workItem, + CancellationToken cancellationToken) + { + var actor = _actorFactory.CreatePullRequestActor(PullRequestActorId.Parse(workItem.ActorId)); + await actor.ProcessPendingUpdatesAsync(workItem); + return true; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs new file mode 100644 index 0000000000..d47d2dd24b --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; +using ProductConstructionService.WorkItems; + +#nullable disable +namespace ProductConstructionService.DependencyFlow.WorkItems; + +[DataContract] +public abstract class ActorWorkItem : WorkItem +{ + [DataMember] + public string ActorId { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs new file mode 100644 index 0000000000..a9bc17dc2b --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; +using Maestro.Contracts; + +#nullable disable +namespace ProductConstructionService.DependencyFlow.WorkItems; + +[DataContract] +public class InProgressPullRequest : ActorWorkItem, IPullRequest +{ + [DataMember] + public string Url { get; set; } + + [DataMember] + public bool? CoherencyCheckSuccessful { get; set; } + + [DataMember] + public List CoherencyErrors { get; set; } + + [DataMember] + public MergePolicyCheckResult MergePolicyResult { get; set; } + + [DataMember] + public List ContainedSubscriptions { get; set; } + + [DataMember] + public List Contained { get; set; } + + [DataMember] + public List RequiredUpdates { get; set; } + + [DataMember] + public bool? SourceRepoNotified { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionTriggerWorkItem.cs similarity index 82% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionTriggerWorkItem.cs index 270dc02cf2..d80ebb5fce 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/UpdateSubscriptionWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionTriggerWorkItem.cs @@ -5,7 +5,7 @@ namespace ProductConstructionService.DependencyFlow.WorkItems; -public class UpdateSubscriptionWorkItem : WorkItem +public class SubscriptionTriggerWorkItem : WorkItem { /// /// Subscription that is being triggered. @@ -15,5 +15,5 @@ public class UpdateSubscriptionWorkItem : WorkItem /// /// Build that is being flown. /// - public required int BuildId { get; init; } + public int? BuildId { get; init; } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs new file mode 100644 index 0000000000..f909a5cc55 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; +using Maestro.Contracts; + +#nullable disable +namespace ProductConstructionService.DependencyFlow.WorkItems; + +[DataContract] +public class SubscriptionUpdateWorkItem : ActorWorkItem +{ + [DataMember] + public Guid SubscriptionId { get; set; } + + [DataMember] + public SubscriptionType SubscriptionType { get; set; } + + [DataMember] + public int BuildId { get; set; } + + [DataMember] + public string SourceSha { get; set; } + + [DataMember] + public string SourceRepo { get; set; } + + [DataMember] + public List Assets { get; set; } + + /// + /// If true, this is a coherency update and not driven by specific + /// subscription ids (e.g. could be multiple if driven by a batched subscription) + /// + [DataMember] + public bool IsCoherencyUpdate { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 1f1865e778..bed8c61b99 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -31,7 +31,7 @@ public SubscriptionTriggerer( public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) { - var workItemProducer = _workItemProducerFactory.CreateProducer(); + var workItemProducer = _workItemProducerFactory.CreateProducer(); foreach (var updateSubscriptionWorkItem in await GetSubscriptionsToTrigger(targetUpdateFrequency)) { await workItemProducer.ProduceWorkItemAsync(updateSubscriptionWorkItem); @@ -41,9 +41,9 @@ public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequenc } } - private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) + private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) { - List subscriptionsToTrigger = new(); + List subscriptionsToTrigger = new(); var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) @@ -53,7 +53,7 @@ private async Task> GetSubscriptionsToTrigger(U .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency); var workItemProducer = - _workItemProducerFactory.CreateProducer(); + _workItemProducerFactory.CreateProducer(); foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { Subscription? subscriptionWithBuilds = await _context.Subscriptions @@ -79,7 +79,7 @@ private async Task> GetSubscriptionsToTrigger(U if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) { - subscriptionsToTrigger.Add(new UpdateSubscriptionWorkItem + subscriptionsToTrigger.Add(new SubscriptionUpdateWorkItem { BuildId = latestBuildInTargetChannel.Id, SubscriptionId = subscription.Id diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs index a0400338e1..d315cdd63c 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManager.cs @@ -7,9 +7,9 @@ namespace ProductConstructionService.WorkItems; public interface IReminderManager where T : WorkItem { - Task RegisterReminderAsync(T reminder, TimeSpan visibilityTimeout); + Task SetReminderAsync(T reminder, TimeSpan dueTime); - Task UnregisterReminderAsync(); + Task UnsetReminderAsync(); } public class ReminderManager : IReminderManager where T : WorkItem @@ -26,14 +26,14 @@ public ReminderManager( _receiptCache = cacheFactory.Create($"ReminderReceipt_{key}"); } - public async Task RegisterReminderAsync(T payload, TimeSpan visibilityTimeout) + public async Task SetReminderAsync(T payload, TimeSpan visibilityTimeout) { var client = _workItemProducerFactory.CreateProducer(); var sendReceipt = await client.ProduceWorkItemAsync(payload, visibilityTimeout); await _receiptCache.SetAsync(new ReminderArguments(sendReceipt.PopReceipt, sendReceipt.MessageId), visibilityTimeout); } - public async Task UnregisterReminderAsync() + public async Task UnsetReminderAsync() { var receipt = await _receiptCache.TryDeleteAsync(); if (receipt == null) diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs index ec3afd0b5a..90bd696a0c 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/ReminderManagerFactory.cs @@ -23,6 +23,7 @@ public ReminderManagerFactory(IWorkItemProducerFactory workItemProducerFactory, public IReminderManager CreateReminderManager(string key) where T : WorkItem { + key = $"{typeof(T).Name}_{key}"; return new ReminderManager(_workItemProducerFactory, _cacheFactory, key); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs index 780bceece2..0df30f438b 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs @@ -35,6 +35,7 @@ public static void AddWorkItemQueues(this IHostApplicationBuilder builder, Defau builder.Services.Configure( builder.Configuration.GetSection(WorkItemConsumerOptions.ConfigurationKey)); builder.Services.AddHostedService(); + builder.Services.AddTransient(); } public static void AddWorkItemProducerFactory(this IHostApplicationBuilder builder, DefaultAzureCredential credential) diff --git a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs index a786d7b6e5..f99ba15426 100644 --- a/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/SubscriptionsController20200220Tests.cs @@ -454,9 +454,9 @@ private static class TestDataConfiguration public static void Dependencies(IServiceCollection collection) { var mockWorkItemProducerFactory = new Mock(); - var mockUpdateSubscriptionWorkItemProducer = new Mock>(); + var mockUpdateSubscriptionWorkItemProducer = new Mock>(); var mockBuildCoherencyInfoWorkItem = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); + mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockUpdateSubscriptionWorkItemProducer.Object); mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockBuildCoherencyInfoWorkItem.Object); collection.AddLogging(l => l.AddProvider(new NUnitLogger())); collection.AddSingleton(new HostingEnvironment diff --git a/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs new file mode 100644 index 0000000000..289bad6799 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs @@ -0,0 +1,149 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Storage.Queues.Models; +using FluentAssertions; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Logging; +using Microsoft.DotNet.Kusto; +using Microsoft.DotNet.Services.Utility; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.Services.Common; +using Moq; +using NUnit.Framework; +using ProductConstructionService.Common; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture] +internal abstract class ActorTests : TestsWithServices +{ + protected const string AssetFeedUrl = "https://source.feed/index.json"; + protected const string SourceBranch = "source.branch"; + protected const string SourceRepo = "source.repo"; + protected const string TargetRepo = "target.repo"; + protected const string TargetBranch = "target.branch"; + protected const string NewBuildNumber = "build.number"; + protected const string NewCommit = "sha2"; + + protected Dictionary ExpectedActorState { get; private set; } = null!; + protected Dictionary ExpectedReminders { get; private set; } = null!; + + protected MockReminderManagerFactory Reminders { get; private set; } = null!; + protected MockRedisCacheFactory RedisCache { get; private set; } = null!; + + protected Mock RemoteFactory { get; private set; } = null!; + protected Dictionary> DarcRemotes { get; private set; } = null!; + protected Mock UpdateResolver { get; private set; } = null!; + protected Mock MergePolicyEvaluator { get; private set; } = null!; + + protected List CodeFlowWorkItemsProduced { get; private set; } = null!; + + + protected override void RegisterServices(IServiceCollection services) + { + base.RegisterServices(services); + services.AddDependencyFlowProcessors(); + services.AddSingleton(RedisCache); + services.AddSingleton(Reminders); + services.AddOperationTracking(_ => { }); + services.AddSingleton(); + services.AddSingleton(Mock.Of()); + services.AddSingleton(Mock.Of()); + services.AddSingleton(RemoteFactory.Object); + services.AddSingleton(MergePolicyEvaluator.Object); + services.AddSingleton(UpdateResolver.Object); + services.AddLogging(); + + // TODO (https://github.com/dotnet/arcade-services/issues/3866): Can be removed once we execute code flow directly + // (when we remove producer factory from the constructor) + Mock workItemProducerFactoryMock = new(); + Mock> workItemProducerMock = new(); + workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) + .ReturnsAsync(QueuesModelFactory.SendReceipt("message", DateTimeOffset.Now, DateTimeOffset.Now, "popReceipt", DateTimeOffset.Now)) + .Callback((item, _) => CodeFlowWorkItemsProduced.Add(item)); + workItemProducerFactoryMock.Setup(w => w.CreateProducer()) + .Returns(workItemProducerMock.Object); + services.AddSingleton(workItemProducerFactoryMock.Object); + + RemoteFactory + .Setup(f => f.GetRemoteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string repo, ILogger logger) => + DarcRemotes.GetOrAddValue(repo, () => CreateMock()).Object); + } + + [SetUp] + public void ActorTests_SetUp() + { + ExpectedActorState = []; + ExpectedReminders = []; + RedisCache = new(); + Reminders = new(); + RemoteFactory = new(); + DarcRemotes = new() + { + [TargetRepo] = new Mock() + }; + MergePolicyEvaluator = new(); + UpdateResolver = new(); + CodeFlowWorkItemsProduced = []; + } + + [TearDown] + public void ActorTests_TearDown() + { + Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders, options => options.ExcludingProperties()); + RedisCache.Data.Should().BeEquivalentTo(ExpectedActorState); + } + + protected void SetReminder(Subscription subscription, T reminder) where T : WorkItem + { + Reminders.Reminders[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = reminder; + } + + protected void RemoveReminder(Subscription subscription) where T : WorkItem + { + Reminders.Reminders.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + } + + protected void SetState(Subscription subscription, T state) where T : class + { + RedisCache.Data[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = state; + } + + protected void RemoveState(Subscription subscription) where T : class + { + RedisCache.Data.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + } + + protected void SetExpectedReminder(Subscription subscription, T reminder) where T : WorkItem + { + ExpectedReminders[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = reminder; + } + + protected void RemoveExpectedReminder(Subscription subscription) where T : WorkItem + { + ExpectedReminders.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + } + + protected void SetExpectedState(Subscription subscription, T state) where T : class + { + ExpectedActorState[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = state; + } + + protected void RemoveExpectedState(Subscription subscription) where T : class + { + ExpectedActorState.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + } + + protected static PullRequestActorId GetPullRequestActorId(Subscription subscription) + { + return subscription.PolicyObject.Batchable + ? new BatchedPullRequestActorId(subscription.TargetRepository, subscription.TargetBranch) + : new NonBatchedPullRequestActorId(subscription.Id); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/Disposable.cs b/test/ProductConstructionService.DependencyFlow.Tests/Disposable.cs new file mode 100644 index 0000000000..90abfd8892 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/Disposable.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal class Disposable : IDisposable +{ + private Action _dispose; + + public static IDisposable Create(Action dispose) + { + return new Disposable(dispose); + } + + private Disposable(Action dispose) + { + _dispose = dispose; + } + + public void Dispose() + { + Interlocked.Exchange(ref _dispose!, null)?.Invoke(); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCache.cs b/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCache.cs new file mode 100644 index 0000000000..4fb6c63b02 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCache.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal abstract class MockRedisCache +{ +} + +internal class MockRedisCache + : MockRedisCache, IRedisCache where T : class +{ + private readonly Dictionary _data = []; + private readonly string _key; + + public IReadOnlyDictionary Data => _data; + + public MockRedisCache(string key, Dictionary data) + { + _key = typeof(T).Name + "_" + key; + _data = data; + } + + public Task SetAsync(T value, TimeSpan? expiration = null) + { + _data[_key] = value; + return Task.CompletedTask; + } + + public Task TryDeleteAsync() + { + _data.Remove(_key, out object? value); + + if (value is null) + { + return Task.FromResult(default(T?)); + } + else + { + return Task.FromResult((T?)value); + } + } + + public Task TryGetStateAsync() + { + return _data.TryGetValue(_key, out object? value) + ? Task.FromResult((T?)value) + : Task.FromResult(default(T?)); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCacheFactory.cs b/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCacheFactory.cs new file mode 100644 index 0000000000..ee72b0c96c --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/MockRedisCacheFactory.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal class MockRedisCacheFactory : IRedisCacheFactory +{ + public Dictionary Data { get; } = []; + + public IRedisCache Create(string key) + { + throw new NotImplementedException(); + } + + public IRedisCache Create(string key) where T : class + { + return new MockRedisCache(key, Data); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs new file mode 100644 index 0000000000..e0a1bbd147 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManager.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal abstract class MockReminderManager +{ +} + +internal class MockReminderManager + : MockReminderManager, IReminderManager where T : WorkItem +{ + public readonly Dictionary Data; + private readonly string _key; + + public MockReminderManager(string key, Dictionary data) + { + _key = key; + Data = data; + } + + public Task SetReminderAsync(T reminder, TimeSpan dueTime) + { + Data[_key] = reminder; + return Task.CompletedTask; + } + + public Task UnsetReminderAsync() + { + Data.Remove(_key); + return Task.CompletedTask; + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManagerFactory.cs b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManagerFactory.cs new file mode 100644 index 0000000000..d052c04fe9 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/MockReminderManagerFactory.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.WorkItems; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal class MockReminderManagerFactory : IReminderManagerFactory +{ + public Dictionary Reminders { get; } = []; + + public IReminderManager CreateReminderManager(string key) where T : WorkItem + { + key = $"{typeof(T).Name}_{key}"; + return new MockReminderManager(key, Reminders); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/ProductConstructionService.DependencyFlow.Tests.csproj b/test/ProductConstructionService.DependencyFlow.Tests/ProductConstructionService.DependencyFlow.Tests.csproj new file mode 100644 index 0000000000..4fd341b593 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/ProductConstructionService.DependencyFlow.Tests.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs new file mode 100644 index 0000000000..2564a3ee29 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs @@ -0,0 +1,316 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Text; +using FluentAssertions; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Maestro.Client.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.Services.Common; +using Moq; +using NUnit.Framework; +using ProductConstructionService.DependencyFlow.WorkItems; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture] +internal class PullRequestBuilderTests : SubscriptionOrPullRequestActorTests +{ + private Dictionary> _darcRemotes = null!; + private Mock _remoteFactory = null!; + private Mock _barClient = null!; + + [SetUp] + public void PullRequestBuilderTests_SetUp() + { + _darcRemotes = new() + { + [TargetRepo] = new Mock() + }; + _remoteFactory = new Mock(MockBehavior.Strict); + _barClient = new Mock(MockBehavior.Strict); + } + + protected override void RegisterServices(IServiceCollection services) + { + _remoteFactory.Setup(f => f.GetRemoteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync( + (string repo, ILogger logger) => + _darcRemotes.GetOrAddValue(repo, () => CreateMock()).Object); + + services.AddSingleton(_remoteFactory.Object); + services.AddSingleton(_barClient.Object); + + base.RegisterServices(services); + } + + [Test] + public async Task ShouldReturnCalculateCorrectPRDescriptionWhenCoherencyUpdate() + { + var build = GivenANewBuildId(101, "abc1"); + SubscriptionUpdateWorkItem update = GivenSubscriptionUpdate(true, build.Id, "11111111-1111-1111-1111-111111111111"); + List deps = GivenDependencyUpdates('a', build.Id); + + var description = await GeneratePullRequestDescription([(update, deps)]); + description.ToString().Should().Contain(BuildCorrectPRDescriptionWhenCoherencyUpdate(deps)); + } + + [Test] + public async Task ShouldReturnCalculateCorrectPRDescriptionWhenNonCoherencyUpdate() + { + var build1 = GivenANewBuildId(101, "abc1"); + var build2 = GivenANewBuildId(102, "def2"); + SubscriptionUpdateWorkItem update1 = GivenSubscriptionUpdate(false, build1.Id, "11111111-1111-1111-1111-111111111111"); + SubscriptionUpdateWorkItem update2 = GivenSubscriptionUpdate(false, build2.Id, "22222222-2222-2222-2222-222222222222"); + List deps1 = GivenDependencyUpdates('a', build1.Id); + List deps2 = GivenDependencyUpdates('b', build2.Id); + + var description = await GeneratePullRequestDescription([(update1, deps1), (update2, deps2)]); + + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps1, 1)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps2, 3)); + } + + [Test] + public async Task ShouldReturnCalculateCorrectPRDescriptionWhenUpdatingExistingPR() + { + var build1 = GivenANewBuildId(101, "abc1"); + var build2 = GivenANewBuildId(102, "def2"); + var build3 = GivenANewBuildId(103, "gha3"); + SubscriptionUpdateWorkItem update1 = GivenSubscriptionUpdate(false, build1.Id, "11111111-1111-1111-1111-111111111111"); + SubscriptionUpdateWorkItem update2 = GivenSubscriptionUpdate(false, build2.Id, "22222222-2222-2222-2222-222222222222"); + SubscriptionUpdateWorkItem update3 = GivenSubscriptionUpdate(false, build3.Id, "33333333-3333-3333-3333-333333333333"); + List deps1 = GivenDependencyUpdates('a', build1.Id); + List deps2 = GivenDependencyUpdates('b', build2.Id); + List deps3 = GivenDependencyUpdates('c', build3.Id); + + var description = await GeneratePullRequestDescription([(update1, deps1), (update2, deps2), (update3, deps3)]); + + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps1, 1)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps2, 3)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps1, 1)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps2, 3)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps3, 5)); + + List deps22 = GivenDependencyUpdates('d', build3.Id); + + description = await GeneratePullRequestDescription([(update2, deps22)], description); + + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps1, 1)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps3, 5)); + description.Should().NotContain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps2, 3)); + description.Should().Contain(BuildCorrectPRDescriptionWhenNonCoherencyUpdate(deps22, 7)); + } + + [Test] + public async Task ShouldReturnCorrectPRDescriptionForCodeEnabledSubscription() + { + Build build = GivenANewBuildId(101, "abc1"); + build.GitHubBranch = "main"; + build.AzureDevOpsBuildNumber = "20230205.2"; + SubscriptionUpdateWorkItem update = GivenSubscriptionUpdate(false, build.Id, guid: "11111111-1111-1111-1111-111111111111"); + update.SubscriptionType = SubscriptionType.DependenciesAndSources; + + string? description = null; + await Execute( + async context => + { + var builder = ActivatorUtilities.CreateInstance(context); + description = await builder.GenerateCodeFlowPRDescriptionAsync(update); + }); + + description.Should().Be( + $""" + [marker]: <> (Begin:11111111-1111-1111-1111-111111111111) + + This pull request is bringing source changes from **The best repo**. + + - **Subscription**: 11111111-1111-1111-1111-111111111111 + - **Build**: 20230205.2 + - **Date Produced**: {build.DateProduced.ToUniversalTime():MMMM d, yyyy h:mm:ss tt UTC} + - **Commit**: abc1 + - **Branch**: main + + [marker]: <> (End:11111111-1111-1111-1111-111111111111) + """); + } + + private const string RegexTestString1 = """ + [2]:qqqq + qqqqq + qqqq + [42]:qq + [2q]:qq + [123] + qq[234]:qq + [345]:qq + """; + + private const string RegexTestString2 = ""; + private const string RegexTestString3 = """ + this + string + shouldn't + have + any + matches + """; + + private const string RegexTestString4 = """ + [1]:q + [2]:1 + [3]:q + [4]:q + """; + private static readonly object[] RegexTestCases = + [ + new object[] { RegexTestString1, 43}, + new object[] { RegexTestString2, 1}, + new object[] { RegexTestString3, 1}, + new object [] { RegexTestString4, 5}, + ]; + + [TestCaseSource(nameof(RegexTestCases))] + public void ShouldReturnCorrectMaximumIndex(string str, int expectedResult) + { + PullRequestBuilder.GetStartingReferenceId(str).Should().Be(expectedResult); + } + + private List GivenDependencyUpdates(char version, int buildId) + { + List dependencies = + [ + new DependencyUpdate + { + From = new DependencyDetail + { + Name = $"from dependency name 1{version}", + Version = $"1.0.0{version}", + CoherentParentDependencyName = $"from parent name 1{version}", + Commit = $"{version} commit from 1" + }, + To = new DependencyDetail + { + Name = $"to dependency name 1{version}", + Version = $"1.0.0{version}", + CoherentParentDependencyName = $"from parent name 1{version}", + RepoUri = "https://amazing_uri.com", + Commit = $"{version} commit to 1", + } + }, + new DependencyUpdate + { + From = new DependencyDetail + { + Name = $"from dependency name 2{version}", + Version = $"1.0.0{version}", + CoherentParentDependencyName = $"from parent name 2{version}", + Commit = $"{version} commit from 2" + }, + To = new DependencyDetail + { + Name = $"to dependency name 2{version}", + Version = $"1.0.0{version}", + CoherentParentDependencyName = $"from parent name 2{version}", + RepoUri = "https://amazing_uri.com", + Commit = $"{version} commit to 2" + } + } + ]; + + foreach (var dependency in dependencies.SelectMany(d => new[] { d.From, d.To })) + { + _barClient + .Setup(x => x.GetAssetsAsync(dependency.Name, dependency.Version, null, null)) + .ReturnsAsync( + [ + new Asset( + 1, + buildId, + false, + dependency.Name, + dependency.Version, + ImmutableList.Empty) + ]); + } + + return dependencies; + } + + private static SubscriptionUpdateWorkItem GivenSubscriptionUpdate(bool isCoherencyUpdate, int buildId, string guid) => new() + { + IsCoherencyUpdate = isCoherencyUpdate, + SourceRepo = "The best repo", + SubscriptionId = new Guid(guid), + BuildId = buildId, + }; + + private static string BuildCorrectPRDescriptionWhenCoherencyUpdate(List deps) + { + var stringBuilder = new StringBuilder(); + foreach (DependencyUpdate dep in deps) + { + stringBuilder.AppendLine($" - **{dep.To.Name}**: from {dep.From.Version} to {dep.To.Version} (parent: {dep.To.CoherentParentDependencyName})"); + } + return stringBuilder.ToString(); + } + + private static string BuildCorrectPRDescriptionWhenNonCoherencyUpdate(List deps, int startingId) + { + var builder = new StringBuilder(); + List urls = []; + for (var i = 0; i < deps.Count; i++) + { + urls.Add(PullRequestBuilder.GetChangesURI(deps[i].To.RepoUri, deps[i].From.Commit, deps[i].To.Commit)); + builder.AppendLine($" - **{deps[i].To.Name}**: [from {deps[i].From.Version} to {deps[i].To.Version}][{startingId + i}]"); + } + builder.AppendLine(); + for (var i = 0; i < urls.Count; i++) + { + builder.AppendLine($"[{i + startingId}]: {urls[i]}"); + } + return builder.ToString(); + } + + private async Task GeneratePullRequestDescription( + List<(SubscriptionUpdateWorkItem update, List deps)> updates, + string? originalDescription = null) + { + string description = null!; + await Execute( + async context => + { + var builder = ActivatorUtilities.CreateInstance(context); + description = await builder.CalculatePRDescriptionAndCommitUpdatesAsync( + updates, + originalDescription, + TargetRepo, + "new-branch"); ; + }); + + return description; + } + + private Build GivenANewBuildId(int id, string sha) + { + Build build = new( + id: id, + dateProduced: DateTimeOffset.Now, + staleness: 0, + released: false, + stable: false, + commit: sha, + channels: ImmutableList.Create(), + assets: ImmutableList.Create(), + dependencies: ImmutableList.Create(), + incoherencies: ImmutableList.Create()); + + _barClient + .Setup(x => x.GetBuildAsync(id)) + .ReturnsAsync(build); + + return build; + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs new file mode 100644 index 0000000000..ba70c8ce79 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs @@ -0,0 +1,280 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using FluentAssertions; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using NUnit.Framework; +using ProductConstructionService.DependencyFlow.WorkItems; +using ClientModels = Microsoft.DotNet.Maestro.Client.Models; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture] +internal class PullRequestPolicyFailureNotifierTests +{ + private const string FakeOrgName = "orgname"; + private const string FakeRepoName = "reponame"; + + protected Mock BarClient = null!; + protected Mock GitRepo = null!; + protected Mock LocalGitClient = null!; + protected Mock RemoteFactory = null!; + protected Mock Env = null!; + protected Mock GithubClient = null!; + protected Mock GitHubTokenProvider = null!; + protected Mock GitHubClientFactory = null!; + protected IServiceScope Scope = null!; + protected Remote MockRemote = null!; + protected ServiceProvider Provider = null!; + protected List FakeSubscriptions = null!; + private Dictionary _prCommentsMade = []; + + [SetUp] + public void PullRequestActorTests_SetUp() + { + _prCommentsMade = []; + + var services = new ServiceCollection(); + FakeSubscriptions = GenerateFakeSubscriptionModels(); + + Env = new Mock(MockBehavior.Strict); + services.AddSingleton(Env.Object); + GithubClient = new Mock(); + GitHubTokenProvider = new Mock(MockBehavior.Strict); + GitHubTokenProvider.Setup(x => x.GetTokenForRepository(It.IsAny())).ReturnsAsync("doesnotmatter"); + GitHubClientFactory = new Mock(); + GitHubClientFactory.Setup(x => x.CreateGitHubClient(It.IsAny())) + .Returns(GithubClient.Object); + GithubClient.Setup(ghc => ghc.Issue.Comment.Create( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(new Octokit.IssueComment()) + .Callback(RecordGitHubClientComment); + + services.AddLogging(); + + BarClient = new Mock(MockBehavior.Strict); + BarClient.Setup(b => b.GetSubscriptionAsync(It.IsAny())) + .Returns((Guid subscriptionToFind) => + { + return Task.FromResult( + (from subscription in FakeSubscriptions + where subscription.Id.Equals(subscriptionToFind) + select subscription).FirstOrDefault()); + }); + + GitRepo = new Mock(MockBehavior.Strict); + GitRepo.Setup(g => g.GetPullRequestChecksAsync(It.IsAny())) + .Returns((string fakePrsUrl) => + { + List checksToReturn = + [ + new Check(CheckState.Failure, "Some Maestro Policy", "", true), + new Check(CheckState.Error, "Some Other Maestro Policy", "", true), + ]; + + if (fakePrsUrl.EndsWith("/12345")) + { + checksToReturn.Add(new Check(CheckState.Failure, "Important PR Check", "", false)); + } + + return Task.FromResult((IList)checksToReturn); + }); + + MockRemote = new Remote(GitRepo.Object, new VersionDetailsParser(), NullLogger.Instance); + + RemoteFactory = new Mock(MockBehavior.Strict); + RemoteFactory.Setup(m => m.GetRemoteAsync(It.IsAny(), It.IsAny())).ReturnsAsync(MockRemote); + + Provider = services.BuildServiceProvider(); + Scope = Provider.CreateScope(); + } + + [TestCase()] + public async Task NotifyACheckFailed() + { + // Happy Path: Successfully create a comment, try to do it again, ensure it does not happen. + var testObject = GetInstance(); + InProgressPullRequest prToTag = GetInProgressPullRequest("https://api.github.com/repos/orgname/reponame/pulls/12345"); + + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + + prToTag.SourceRepoNotified.Should().BeTrue(); + + // Second time; no second comment should be made. (If it were made, it'd throw) + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + _prCommentsMade.Count.Should().Be(1); + // Spot check some values + _prCommentsMade[$"{FakeOrgName}/{FakeRepoName}/12345"].Should().Contain( + $"Notification for subscribed users from https://github.com/{FakeOrgName}/source-repo1"); + foreach (var individual in FakeSubscriptions[0].PullRequestFailureNotificationTags.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + // Make sure normalization happens; test includes a user without @. + var valueToCheck = individual; + if (!individual.StartsWith('@')) + valueToCheck = $"@{valueToCheck}"; + + _prCommentsMade[$"{FakeOrgName}/{FakeRepoName}/12345"].Should().Contain(valueToCheck); + } + } + + [TestCase()] + public async Task MultipleBuildsIncluded() + { + // Scenario where a bunch of commits / builds are included in the same non-batched, single subscription PR + var testObject = GetInstance(); + InProgressPullRequest prToTag = GetInProgressPullRequest("https://api.github.com/repos/orgname/reponame/pulls/12345", 2); + + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + + prToTag.SourceRepoNotified.Should().BeTrue(); + + // Second time; no second comment should be made. (If it were made, it'd throw) + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + _prCommentsMade.Count.Should().Be(1); + // Spot check some values + _prCommentsMade[$"{FakeOrgName}/{FakeRepoName}/12345"].Should().Contain( + $"Notification for subscribed users from https://github.com/{FakeOrgName}/source-repo1"); + foreach (var individual in FakeSubscriptions[0].PullRequestFailureNotificationTags.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + // Make sure normalization happens; test includes a user without @. + var valueToCheck = individual; + if (!individual.StartsWith('@')) + valueToCheck = $"@{valueToCheck}"; + + _prCommentsMade[$"{FakeOrgName}/{FakeRepoName}/12345"].Should().Contain(valueToCheck); + } + } + + [TestCase()] + public async Task OnlyMaestroChecksHaveFailedOrErrored() + { + // Checks like "all checks succeeded" stay in a failed state until all the other checks, err, succeed. + // Ensure we don't just go tag everyone's automerge PRs. + var testObject = GetInstance(); + InProgressPullRequest prToTag = GetInProgressPullRequest("https://api.github.com/repos/orgname/reponame/pulls/67890", 1); + + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + + prToTag.SourceRepoNotified.Should().BeFalse(); + _prCommentsMade.Count.Should().Be(0); + } + + [TestCase()] + public async Task NoContactAliasesProvided() + { + // "Do nothing" Path: Just don't blow up when a subscription object has no tags. + var testObject = GetInstance(); + InProgressPullRequest prToTag = GetInProgressPullRequestWithoutTags("https://api.github.com/repos/orgname/reponame/pulls/23456"); + await testObject.TagSourceRepositoryGitHubContactsAsync(prToTag); + prToTag.SourceRepoNotified.Should().BeFalse(); + _prCommentsMade.Count.Should().Be(0); + } + + #region Test Helpers + + private InProgressPullRequest GetInProgressPullRequest(string url, int containedSubscriptionCount = 1) + { + var containedSubscriptions = new List(); + + for (var i = 0; i < containedSubscriptionCount; i++) + { + containedSubscriptions.Add(new SubscriptionPullRequestUpdate() + { + BuildId = 10000 + i, + SubscriptionId = FakeSubscriptions[i].Id + }); + } + // For the purposes of testing this class, we only need to fill out + // the "Url" and ContainedSubscriptions fields in InProgressPullRequestObjects + return new InProgressPullRequest() + { + Url = url, + ContainedSubscriptions = containedSubscriptions, + SourceRepoNotified = false + }; + } + + private InProgressPullRequest GetInProgressPullRequestWithoutTags(string url) + { + var containedSubscriptions = new List(); + + var tagless = FakeSubscriptions.Where(f => string.IsNullOrEmpty(f.PullRequestFailureNotificationTags)).First(); + containedSubscriptions.Add(new SubscriptionPullRequestUpdate() + { + BuildId = 12345, + SubscriptionId = tagless.Id + }); + + return new InProgressPullRequest() + { + Url = url, + ContainedSubscriptions = containedSubscriptions, + SourceRepoNotified = false + }; + } + + private void RecordGitHubClientComment(string owner, string repo, int prIssue, string comment) + { + lock (_prCommentsMade) + { + // No need to check for existence; if the same comment gets added twice, it'd be a bug + _prCommentsMade.Add($"{owner}/{repo}/{prIssue}", comment); + } + } + + public IPullRequestPolicyFailureNotifier GetInstance() => new PullRequestPolicyFailureNotifier( + GitHubTokenProvider.Object, + GitHubClientFactory.Object, + RemoteFactory.Object, + BarClient.Object, + Scope.ServiceProvider.GetRequiredService>()); + + private static List GenerateFakeSubscriptionModels() => + [ + new ClientModels.Subscription( + new Guid("35684498-9C08-431F-8E66-8242D7C38598"), + true, + false, + $"https://github.com/{FakeOrgName}/source-repo1", + $"https://github.com/{FakeOrgName}/dest-repo", + "fakebranch", + null, + null, + "@notifiedUser1;@notifiedUser2;userWithoutAtSign;", + excludedAssets: ImmutableList.Empty), + new ClientModels.Subscription( + new Guid("80B3B6EE-4C9B-46AC-B275-E016E0D5AF41"), + true, + false, + $"https://github.com/{FakeOrgName}/source-repo2", + $"https://github.com/{FakeOrgName}/dest-repo", + "fakebranch", + null, + null, + "@notifiedUser3;@notifiedUser4", + excludedAssets: ImmutableList.Empty), + new ClientModels.Subscription( + new Guid("1802E0D2-D6BF-4A14-BF4C-B2A292739E59"), + true, + false, + $"https://github.com/{FakeOrgName}/source-repo2", + $"https://github.com/{FakeOrgName}/dest-repo", + "fakebranch", + null, + null, + string.Empty, + excludedAssets: ImmutableList.Empty) + ]; + + #endregion +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs new file mode 100644 index 0000000000..82f4b6d070 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Maestro.Data.Models; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.Services.Common; +using Moq; +using NUnit.Framework; +using Asset = Maestro.Contracts.Asset; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture, NonParallelizable] +internal class SubscriptionActorTests : SubscriptionOrPullRequestActorTests +{ + protected Dictionary> PullRequestActors { get; private set; } = []; + + [SetUp] + public void SubscriptionActorTests_SetUp() + { + PullRequestActors.Clear(); + } + + protected override void RegisterServices(IServiceCollection services) + { + base.RegisterServices(services); + + var actorFactory = new Mock(); + + actorFactory + .Setup(l => l.CreatePullRequestActor(It.IsAny())) + .Returns((PullRequestActorId actorId) => + { + Mock mock = PullRequestActors.GetOrAddValue( + actorId, + () => CreateMock()); + return mock.Object; + }); + + services.AddSingleton(actorFactory.Object); + } + + internal async Task WhenUpdateAsyncIsCalled(Subscription forSubscription, Build andForBuild) + { + await Execute( + async provider => + { + ISubscriptionActor actor = ActivatorUtilities.CreateInstance(provider, forSubscription.Id); + await actor.UpdateSubscriptionAsync(andForBuild.Id); + }); + } + + private void ThenUpdateAssetsAsyncShouldHaveBeenCalled(ActorId forActor, Build withBuild) + { + var updatedAssets = new List>(); + PullRequestActors.Should().ContainKey(forActor) + .WhoseValue.Verify( + a => a.UpdateAssetsAsync(Subscription.Id, SubscriptionType.Dependencies, withBuild.Id, SourceRepo, NewCommit, Capture.In(updatedAssets))); + + updatedAssets.Should().BeEquivalentTo( + new List> + { + withBuild.Assets + .Select(a => new Asset + { + Name = a.Name, + Version = a.Version + }) + .ToList() + }); + } + + [Test] + public async Task BatchableEveryBuildSubscription() + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + await WhenUpdateAsyncIsCalled(Subscription, b); + ThenUpdateAssetsAsyncShouldHaveBeenCalled( + new BatchedPullRequestActorId(Subscription.TargetRepository, Subscription.TargetBranch), + b); + } + + [Test] + public async Task NotBatchableEveryBuildSubscription() + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + await WhenUpdateAsyncIsCalled(Subscription, b); + ThenUpdateAssetsAsyncShouldHaveBeenCalled(new NonBatchedPullRequestActorId(Subscription.Id), b); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs new file mode 100644 index 0000000000..fd593dc18e --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture] +internal abstract class SubscriptionOrPullRequestActorTests : ActorTests +{ + protected List> ContextUpdates = null!; + protected List AfterDbUpdateActions = null!; + protected Mock HostingEnvironment = null!; + protected Channel Channel = null!; + protected DefaultChannel DefaultChannel = null!; + protected Subscription Subscription = null!; + + [SetUp] + public void SubscriptionOrPullRequestActorTests_SetUp() + { + ContextUpdates = []; + AfterDbUpdateActions = []; + HostingEnvironment = CreateMock(); + } + + protected override void RegisterServices(IServiceCollection services) + { + base.RegisterServices(services); + + services.AddSingleton(HostingEnvironment.Object); + services.AddBuildAssetRegistry(options => + { + options.UseInMemoryDatabase("BuildAssetRegistry"); + options.EnableServiceProviderCaching(false); + }); + } + + protected override async Task BeforeExecute(IServiceProvider context) + { + var dbContext = context.GetRequiredService(); + foreach (Action update in ContextUpdates) + { + update(dbContext); + } + + await dbContext.SaveChangesAsync(); + + foreach (Action update in AfterDbUpdateActions) + { + update(); + } + } + + internal void GivenADefaultChannel(bool enabled) + { + DefaultChannel = new DefaultChannel + { + Branch = SourceBranch, + Channel = Channel, + ChannelId = Channel.Id, + Enabled = enabled, + Repository = SourceRepo + }; + ContextUpdates.Add(context => context.DefaultChannels.Add(DefaultChannel)); + } + + internal void GivenATestChannel() + { + Channel = new Channel + { + Name = "channel", + Classification = "class" + }; + ContextUpdates.Add(context => context.Channels.Add(Channel)); + } + + internal void GivenASubscription(SubscriptionPolicy policy) + { + Subscription = new Subscription + { + Channel = Channel, + SourceRepository = SourceRepo, + TargetRepository = TargetRepo, + TargetBranch = TargetBranch, + PolicyObject = policy + }; + ContextUpdates.Add(context => context.Subscriptions.Add(Subscription)); + } + + internal void GivenACodeFlowSubscription(SubscriptionPolicy policy) + { + Subscription = new Subscription + { + Channel = Channel, + SourceRepository = SourceRepo, + TargetRepository = TargetRepo, + TargetBranch = TargetBranch, + PolicyObject = policy, + + SourceEnabled = true, + SourceDirectory = "repo", + ExcludedAssets = [new AssetFilter() { Filter = "Excluded.Package" }], + }; + ContextUpdates.Add(context => context.Subscriptions.Add(Subscription)); + } + + internal Build GivenANewBuild(bool addToChannel, (string name, string version, bool nonShipping)[]? assets = null) + { + assets ??= [("quail.eating.ducks", "1.1.0", false), ("quail.eating.ducks", "1.1.0", false), ("quite.expensive.device", "2.0.1", true)]; + var build = new Build + { + GitHubBranch = SourceBranch, + GitHubRepository = SourceRepo, + AzureDevOpsBuildNumber = NewBuildNumber, + AzureDevOpsBranch = SourceBranch, + AzureDevOpsRepository = SourceRepo, + Commit = NewCommit, + DateProduced = DateTimeOffset.UtcNow, + Assets = new List( + assets.Select( + a => new Asset + { + Name = a.name, + Version = a.version, + NonShipping = a.nonShipping, + Locations = + [ + new AssetLocation + { + Location = AssetFeedUrl, + Type = LocationType.NugetFeed + } + ] + })) + }; + ContextUpdates.Add( + context => + { + context.Builds.Add(build); + if (addToChannel) + { + context.BuildChannels.Add( + new BuildChannel + { + Build = build, + Channel = Channel, + DateTimeAdded = DateTimeOffset.UtcNow + }); + } + }); + return build; + } + + protected PullRequestActorId GetPullRequestActorId() + { + return Subscription.PolicyObject.Batchable + ? new BatchedPullRequestActorId(Subscription.TargetRepository, Subscription.TargetBranch) + : new NonBatchedPullRequestActorId(Subscription.Id); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/TestsWithMocks.cs b/test/ProductConstructionService.DependencyFlow.Tests/TestsWithMocks.cs new file mode 100644 index 0000000000..d8ce83b8b7 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/TestsWithMocks.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Moq; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture] +public class TestsWithMocks +{ + private VerifyableMockRepository _mocks = null!; + + [SetUp] + public void TestsWithMocks_SetUp() + { + _mocks = new VerifyableMockRepository(MockBehavior.Loose); + } + + [TearDown] + public void TestsWithMocks_TearDown() + { + _mocks.VerifyNoUnverifiedCalls(); + } + + protected Mock CreateMock(MockBehavior behavior = MockBehavior.Default) where T : class + { + return _mocks.Create(); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/TestsWithServices.cs b/test/ProductConstructionService.DependencyFlow.Tests/TestsWithServices.cs new file mode 100644 index 0000000000..ca4f2addf6 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/TestsWithServices.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.DotNet.ServiceFabric.ServiceHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; + +namespace ProductConstructionService.DependencyFlow.Tests; + +public abstract class TestsWithServices : TestsWithMocks +{ + protected virtual void RegisterServices(IServiceCollection services) + { + } + + protected virtual Task BeforeExecute(IServiceProvider serviceScope) + { + return Task.CompletedTask; + } + + protected async Task Execute(Func run) + { + var services = new ServiceCollection(); + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "XUNIT"); + services.TryAddSingleton(typeof(IActorProxyFactory<>), typeof(ActorProxyFactory<>)); + services.AddLogging(l => l.AddProvider(new NUnitLogger())); + RegisterServices(services); + using (ServiceProvider container = services.BuildServiceProvider()) + using (IServiceScope scope = container.GetRequiredService().CreateScope()) + { + await BeforeExecute(scope.ServiceProvider); + await run(scope.ServiceProvider); + } + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/VerifyableMockRepository.cs b/test/ProductConstructionService.DependencyFlow.Tests/VerifyableMockRepository.cs new file mode 100644 index 0000000000..5e9ba05c65 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/VerifyableMockRepository.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Moq; + +namespace ProductConstructionService.DependencyFlow.Tests; + +public class VerifyableMockRepository : MockRepository +{ + public VerifyableMockRepository(MockBehavior defaultBehavior) : base(defaultBehavior) + { + } + + public void VerifyNoUnverifiedCalls() + { + foreach (dynamic mock in Mocks) + { + mock.VerifyNoOtherCalls(); + } + } +} diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs index 3edb477fa2..61c2fc0b27 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs @@ -21,7 +21,7 @@ public class SubscriptionTriggererTests private BuildAssetRegistryContext? _context; private ServiceProvider? _provider; private IServiceScope _scope = new Mock().Object; - private List _updateSubscriptionWorkItems = new(); + private List _updateSubscriptionWorkItems = []; [SetUp] public void Setup() @@ -30,12 +30,12 @@ public void Setup() _updateSubscriptionWorkItems = new(); Mock workItemProducerFactoryMock = new(); - Mock> workItemProducerMock = new(); + Mock> workItemProducerMock = new(); - workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) + workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) .ReturnsAsync(QueuesModelFactory.SendReceipt("message", DateTimeOffset.Now, DateTimeOffset.Now, "popReceipt", DateTimeOffset.Now)) - .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); - workItemProducerFactoryMock.Setup(w => w.CreateProducer()) + .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); + workItemProducerFactoryMock.Setup(w => w.CreateProducer()) .Returns(workItemProducerMock.Object); services.AddLogging(); From 6e94311ae6a34b5586ae2095046aef589fe591c8 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 28 Aug 2024 00:07:07 +0200 Subject: [PATCH 07/33] Fix data protection related deployment errors (#3883) --- ...-pipelines-product-construction-service.yml | 11 +++++++++++ .../ProductConstructionService/provision.bicep | 18 ++++++++++++++++-- .../Configuration/DataProtection.cs | 3 +-- .../PcsStartup.cs | 6 +++--- .../WorkItemConfiguration.cs | 2 +- 5 files changed, 32 insertions(+), 8 deletions(-) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index a105ccd795..5c0f63f74f 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -65,6 +65,17 @@ stages: - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}: + - task: AzureCLI@2 + inputs: + azureSubscription: $(serviceConnectionName) + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + New-Item -ItemType Directory -Path $(diffFolder) + $before = az containerapp show --name $(containerappName) -g $(resourceGroupName) --output json + Set-Content -Path $(diffFolder)/before.json -Value $before + displayName: Snapshot configuration (before) + - task: AzureCLI@2 name: GetAuthInfo displayName: Get PCS Token diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 309c165cee..1f3e4b8199 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -79,10 +79,10 @@ param longestBuildPathUpdaterIdentityName string = 'LongestBuildPathUpdaterInt' param longestBuildPathUpdaterJobName string = 'longest-path-updater-job-int' @description('Network security group name') -var networkSecurityGroupName = 'product-construction-service-nsg-int' +param networkSecurityGroupName string = 'product-construction-service-nsg-int' @description('Resource group where PCS IP resources will be created') -var infrastructureResourceGroupName = 'product-construction-service-ip-int' +param infrastructureResourceGroupName string = 'product-construction-service-ip-int' resource networkSecurityGroup 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { name: networkSecurityGroupName @@ -424,6 +424,8 @@ var contributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefini var keyVaultReaderRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '21090545-7ca7-4776-b22c-e363652d74d2') // storage account blob contributor var blobContributorRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe') +// Key Vault Crypto User role +var kvCryptoUserRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') // application insights for service logging resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = { @@ -715,6 +717,18 @@ resource secretAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { } } +// allow crypto access to the identity used for the aca's +resource cryptoAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: keyVault // Use when specifying a scope that is different than the deployment scope + name: guid(subscription().id, resourceGroup().id, kvCryptoUserRole) + properties: { + roleDefinitionId: kvCryptoUserRole + principalType: 'ServicePrincipal' + principalId: pcsIdentity.properties.principalId + } +} + + resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { name: storageAccountName location: location diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs index ba9bc9c6ae..27a23c4df8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/DataProtection.cs @@ -13,7 +13,7 @@ internal static class DataProtection private static readonly TimeSpan DataProtectionKeyLifetime = new(days: 240, hours: 0, minutes: 0, seconds: 0); - public static void AddDataProtection(this WebApplicationBuilder builder) + public static void AddDataProtection(this WebApplicationBuilder builder, DefaultAzureCredential credential) { var keyBlobUri = builder.Configuration[DataProtectionKeyBlobUri]; var dataProtectionKeyUri = builder.Configuration[DataProtectionKeyUri]; @@ -26,7 +26,6 @@ public static void AddDataProtection(this WebApplicationBuilder builder) return; } - var credential = new DefaultAzureCredential(); builder.Services.AddDataProtection() .PersistKeysToAzureBlobStorage(new Uri(keyBlobUri), credential) .ProtectKeysWithAzureKeyVault(new Uri(dataProtectionKeyUri), credential) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index e0165bfe30..f4d83a2057 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -146,14 +146,14 @@ internal static async Task ConfigurePcs( string? gitHubToken = builder.Configuration[ConfigurationKeys.GitHubToken]; builder.Services.Configure(ConfigurationKeys.AzureDevOpsConfiguration, (o, s) => s.Bind(o)); - builder.AddDataProtection(); - builder.AddTelemetry(); - DefaultAzureCredential azureCredential = new(new DefaultAzureCredentialOptions { ManagedIdentityClientId = managedIdentityId, }); + builder.AddDataProtection(azureCredential); + builder.AddTelemetry(); + if (addKeyVault) { Uri keyVaultUri = new($"https://{builder.Configuration.GetRequiredValue(ConfigurationKeys.KeyVaultName)}.vault.azure.net/"); diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs index 0df30f438b..150bf9c24d 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemConfiguration.cs @@ -30,7 +30,7 @@ public static void AddWorkItemQueues(this IHostApplicationBuilder builder, Defau builder.Services.AddSingleton(sp => new WorkItemScopeManager(waitForInitialization, sp, sp.GetRequiredService>())); - builder.Configuration[$"{WorkItemConsumerOptions.ConfigurationKey}:${WorkItemQueueNameConfigurationKey}"] = + builder.Configuration[$"{WorkItemConsumerOptions.ConfigurationKey}:{WorkItemQueueNameConfigurationKey}"] = builder.Configuration.GetRequiredValue(WorkItemQueueNameConfigurationKey); builder.Services.Configure( builder.Configuration.GetSection(WorkItemConsumerOptions.ConfigurationKey)); From ad4b4bd2469b713f08419fb9b065223ee84f5a1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 28 Aug 2024 10:53:15 +0200 Subject: [PATCH 08/33] Print direct link to revision logs when deployment fails (#3886) --- ...pipelines-product-construction-service.yml | 8 +++- .../product-construction-service-deploy.ps1 | 42 ++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 5c0f63f74f..02d64b5e50 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -12,6 +12,7 @@ pr: - main variables: +# https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=189 - group: Publish-Build-Assets - name: resourceGroupName value: product-construction-service @@ -20,7 +21,8 @@ variables: - name: diffFolder value: $(Build.ArtifactStagingDirectory)/diff - ${{ if ne(variables['Build.SourceBranch'], 'refs/heads/production') }}: - # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=189 + - name: subscriptionId + value: e6b5f9f5-0ca4-4351-879b-014d78400ec2 - name: containerappName value: product-construction-int - name: containerjobNames @@ -29,6 +31,8 @@ variables: value: productconstructionint - name: containerappEnvironmentName value: product-construction-service-env-int + - name: containerappWorkspaceName + value: product-construction-service-workspace-int - name: dockerRegistryUrl value: productconstructionint.azurecr.io - name: serviceConnectionName @@ -96,8 +100,10 @@ stages: scriptLocation: scriptPath scriptPath: $(Build.SourcesDirectory)/eng/deployment/product-construction-service-deploy.ps1 arguments: > + -subscriptionId $(subscriptionId) -resourceGroupName $(resourceGroupName) -containerappName $(containerappName) + -workspaceName $(containerappWorkspaceName) -newImageTag $(DockerTag.newDockerImageTag) -containerRegistryName $(containerRegistryName) -imageName $(containerName) diff --git a/eng/deployment/product-construction-service-deploy.ps1 b/eng/deployment/product-construction-service-deploy.ps1 index 50fb761df3..1c4ac33fbc 100644 --- a/eng/deployment/product-construction-service-deploy.ps1 +++ b/eng/deployment/product-construction-service-deploy.ps1 @@ -2,8 +2,10 @@ # The script determines the color of the currently active revision, deactivates the old inactive revision, # and deploys the new revision, switching all traffic to it if the health probes pass. param( + [Parameter(Mandatory=$true)][string]$subscriptionId, [Parameter(Mandatory=$true)][string]$resourceGroupName, [Parameter(Mandatory=$true)][string]$containerappName, + [Parameter(Mandatory=$true)][string]$workspaceName, [Parameter(Mandatory=$true)][string]$newImageTag, [Parameter(Mandatory=$true)][string]$containerRegistryName, [Parameter(Mandatory=$true)][string]$imageName, @@ -52,12 +54,39 @@ function StopAndWait([string]$pcsStatusUrl, [string]$pcsStopUrl, [hashtable]$aut return } +function GetLogsLink { + param ( + [string]$revisionName, + [string]$resourceGroup, + [string]$workspaceName, + [string]$subscriptionId + ) + + $query = "ContainerAppConsoleLogs_CL ` +| where RevisionName_s == '$revisionName' ` +| project TimeGenerated, Log_s" + + $bytes = [System.Text.Encoding]::UTF8.GetBytes($query) + $memoryStream = New-Object System.IO.MemoryStream + $compressedStream = New-Object System.IO.Compression.GZipStream($memoryStream, [System.IO.Compression.CompressionMode]::Compress, $true) + + $compressedStream.Write($bytes, 0, $bytes.Length) + $compressedStream.Close() + $memoryStream.Seek(0, [System.IO.SeekOrigin]::Begin) | Out-Null + $data = $memoryStream.ToArray() + $encodedQuery = [Convert]::ToBase64String($data) + $encodedQuery = [System.Web.HttpUtility]::UrlEncode($encodedQuery) + return "https://ms.portal.azure.com#@72f988bf-86f1-41af-91ab-2d7cd011db47/blade/Microsoft_OperationsManagementSuite_Workspace/Logs.ReactView/" + + "resourceId/%2Fsubscriptions%2F$subscriptionId%2FresourceGroups%2F$resourceGroup%2Fproviders%2FMicrosoft.OperationalInsights%2Fworkspaces%2F" + + "$workspaceName/source/LogsBlade.AnalyticsShareLinkToQuery/q/$encodedQuery/timespan/P1D/limit/1000" +} + az extension add --name containerapp --upgrade Write-Host "Fetching all revisions to determine the active label" $containerappTraffic = az containerapp ingress traffic show --name $containerappName --resource-group $resourceGroupName | ConvertFrom-Json # find the currently active revision -$activeRevision = $containerappTraffic | Where-Object { $_.weight -eq 100 } +$activeRevision = $containerappTraffic | Where-Object { $_.weight -eq 100 } Write-Host "Currently active revision: $($activeRevision.revisionName) with label $($activeRevision.label)" @@ -125,7 +154,16 @@ try Write-Host "All traffic has been redirected to label $inactiveLabel" } else { - Write-Warning "New revision is not running. Check revision $newRevisionName logs in the inactive revisions. Deactivating the new revision" + Write-Warning "New revision failed to start. Deactivating the new revision.." + $link = GetLogsLink ` + -revisionName "$newRevisionName" ` + -subscriptionId "$subscriptionId" ` + -resourceGroup "$resourceGroupName" ` + -workspaceName "$workspaceName" + + Write-Host " Check revision $newRevisionName logs in the inactive revision: ` + $link" + az containerapp revision deactivate --revision $newRevisionName --name $containerappName --resource-group $resourceGroupName exit 1 } From 37b21ccf57db3a0e6e63ec9027331a6eb4209725 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 28 Aug 2024 15:45:29 +0200 Subject: [PATCH 09/33] Port `PullRequestActor` to PCS (#3887) Co-authored-by: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> --- .../ValidateCoherencyMergePolicy.cs | 2 +- .../PcsStartup.cs | 2 +- .../BatchedPullRequestActor.cs | 27 +- .../DependencyFlowConfiguration.cs | 1 + .../IPullRequestActor.cs | 2 +- .../NonBatchedPullRequestActor.cs | 92 +- .../PullRequestActor.cs | 955 +++++++++++++++++- .../WorkItems/ActorWorkItem.cs | 2 +- .../WorkItems/InProgressPullRequest.cs | 6 +- .../WorkItems/SubscriptionUpdateWorkItem.cs | 14 +- .../SubscriptionTriggerer.cs | 10 +- .../WorkItem.cs | 2 +- .../ChannelsController20180716Tests.cs | 11 + .../ChannelsController20200220Tests.cs | 5 +- .../ActorTests.cs | 16 +- .../PendingCodeFlowUpdatesTests.cs | 88 ++ .../PendingUpdatePullRequestActorTests.cs | 41 + .../PendingUpdatesTests.cs | 57 ++ .../PullRequestActorTests.cs | 472 +++++++++ .../PullRequestBuilderTests.cs | 11 +- .../PullRequestPolicyFailureNotifierTests.cs | 2 + .../SubscriptionOrPullRequestActorTests.cs | 33 +- .../UpdateAssetsForCodeFlowTests.cs | 194 ++++ .../UpdateAssetsPullRequestActorTests.cs | 32 + .../UpdateAssetsTests.cs | 156 +++ .../SubscriptionTriggererTests.cs | 12 +- 26 files changed, 2171 insertions(+), 74 deletions(-) create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs create mode 100644 test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs diff --git a/src/Maestro/Maestro.MergePolicies/ValidateCoherencyMergePolicy.cs b/src/Maestro/Maestro.MergePolicies/ValidateCoherencyMergePolicy.cs index 23f09e7d55..8720c27426 100644 --- a/src/Maestro/Maestro.MergePolicies/ValidateCoherencyMergePolicy.cs +++ b/src/Maestro/Maestro.MergePolicies/ValidateCoherencyMergePolicy.cs @@ -21,7 +21,7 @@ public override Task EvaluateAsync(IPullRequest pr, return Task.FromResult(Succeed("Coherency check successful.")); var description = new StringBuilder("Coherency update failed for the following dependencies:"); - foreach (CoherencyErrorDetails error in pr.CoherencyErrors ?? Enumerable.Empty()) + foreach (CoherencyErrorDetails error in pr.CoherencyErrors ?? []) { description.Append("\n * ").Append(error.Error); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index f4d83a2057..b1c1eacba7 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -76,7 +76,7 @@ static PcsStartup() { var context = (BuildAssetRegistryContext)entry.Context; ILogger logger = context.GetService>(); - var workItemProducer = context.GetService().CreateProducer(); + var workItemProducer = context.GetService().CreateProducer(); var subscriptionIdGenerator = context.GetService(); BuildChannel entity = entry.Entity; diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs index e611b0c130..26f5a042d7 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs @@ -3,6 +3,10 @@ using Maestro.Data; using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.Logging; +using ProductConstructionService.Common; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.DependencyFlow; @@ -17,8 +21,27 @@ internal class BatchedPullRequestActor : PullRequestActor public BatchedPullRequestActor( BatchedPullRequestActorId id, - BuildAssetRegistryContext context) - : base(id) + IMergePolicyEvaluator mergePolicyEvaluator, + BuildAssetRegistryContext context, + IRemoteFactory remoteFactory, + IActorFactory actorFactory, + ICoherencyUpdateResolver coherencyUpdateResolver, + IPullRequestBuilder pullRequestBuilder, + IRedisCacheFactory cacheFactory, + IReminderManagerFactory reminderManagerFactory, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) + : base( + id, + mergePolicyEvaluator, + remoteFactory, + actorFactory, + coherencyUpdateResolver, + pullRequestBuilder, + cacheFactory, + reminderManagerFactory, + workItemProducerFactory, + logger) { _id = id; _context = context; diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs index f723b6cfa6..d885e19c0c 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs @@ -22,6 +22,7 @@ public static void AddDependencyFlowProcessors(this IServiceCollection services) services.TryAddTransient(); services.TryAddSingleton(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddScoped(); services.AddWorkItemProcessor(); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs index 70a1aabafa..a00ba43d62 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs @@ -7,7 +7,7 @@ namespace ProductConstructionService.DependencyFlow; public interface IPullRequestActor { - Task<(InProgressPullRequest? pr, bool canUpdate)> SynchronizeInProgressPullRequestAsync( + Task SynchronizeInProgressPullRequestAsync( InProgressPullRequest pullRequestCheck); Task ProcessPendingUpdatesAsync( diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs index 6099f784a0..46c2876b7c 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs @@ -1,7 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Maestro.Data; using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.Logging; +using ProductConstructionService.Common; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.DependencyFlow; @@ -11,11 +17,89 @@ namespace ProductConstructionService.DependencyFlow; /// internal class NonBatchedPullRequestActor : PullRequestActor { - public NonBatchedPullRequestActor(NonBatchedPullRequestActorId id) - : base(id) + private readonly Lazy> _lazySubscription; + private readonly NonBatchedPullRequestActorId _id; + private readonly BuildAssetRegistryContext _context; + private readonly IPullRequestPolicyFailureNotifier _pullRequestPolicyFailureNotifier; + + public NonBatchedPullRequestActor( + NonBatchedPullRequestActorId id, + IMergePolicyEvaluator mergePolicyEvaluator, + BuildAssetRegistryContext context, + IRemoteFactory remoteFactory, + IActorFactory actorFactory, + ICoherencyUpdateResolver coherencyUpdateResolver, + IPullRequestBuilder pullRequestBuilder, + IPullRequestPolicyFailureNotifier pullRequestPolicyFailureNotifier, + IRedisCacheFactory cacheFactory, + IReminderManagerFactory reminderManagerFactory, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) + : base( + id, + mergePolicyEvaluator, + remoteFactory, + actorFactory, + coherencyUpdateResolver, + pullRequestBuilder, + cacheFactory, + reminderManagerFactory, + workItemProducerFactory, + logger) + { + _lazySubscription = new Lazy>(RetrieveSubscription); + _id = id; + _context = context; + _pullRequestPolicyFailureNotifier = pullRequestPolicyFailureNotifier; + } + + public Guid SubscriptionId => _id.SubscriptionId; + + private async Task RetrieveSubscription() { + Subscription? subscription = await _context.Subscriptions.FindAsync(SubscriptionId); + if (subscription == null) + { + await _pullRequestCheckReminders.UnsetReminderAsync(); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + return null; + } + + return subscription; } - protected override Task> GetMergePolicyDefinitions() => throw new NotImplementedException(); - protected override Task<(string repository, string branch)> GetTargetAsync() => throw new NotImplementedException(); + private Task GetSubscription() + { + return _lazySubscription.Value; + } + + protected override async Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgressPullRequest pr) + { + await _pullRequestPolicyFailureNotifier.TagSourceRepositoryGitHubContactsAsync(pr); + } + + protected override async Task<(string repository, string branch)> GetTargetAsync() + { + Subscription subscription = await GetSubscription() + ?? throw new SubscriptionException($"Subscription '{SubscriptionId}' was not found..."); + return (subscription.TargetRepository, subscription.TargetBranch); + } + + protected override async Task> GetMergePolicyDefinitions() + { + Subscription? subscription = await GetSubscription(); + return subscription?.PolicyObject?.MergePolicies ?? []; + } + + public override async Task SynchronizeInProgressPullRequestAsync( + InProgressPullRequest pullRequestCheck) + { + Subscription? subscription = await GetSubscription(); + if (subscription == null) + { + return false; + } + + return await base.SynchronizeInProgressPullRequestAsync(pullRequestCheck); + } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs index 33d34acf8c..fe3788f648 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs @@ -1,10 +1,19 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Net; +using Maestro.Contracts; +using Maestro.Data; using Maestro.Data.Models; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.DarcLib; +using Microsoft.Extensions.Logging; +using ProductConstructionService.Common; using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; using Asset = Maestro.Contracts.Asset; +using AssetData = Microsoft.DotNet.Maestro.Client.Models.AssetData; namespace ProductConstructionService.DependencyFlow; @@ -14,15 +23,51 @@ namespace ProductConstructionService.DependencyFlow; /// internal abstract class PullRequestActor : IPullRequestActor { + private static readonly TimeSpan DefaultReminderDuration = TimeSpan.FromMinutes(5); + private readonly PullRequestActorId _id; + private readonly IMergePolicyEvaluator _mergePolicyEvaluator; + private readonly IRemoteFactory _remoteFactory; + private readonly IActorFactory _actorFactory; + private readonly ICoherencyUpdateResolver _coherencyUpdateResolver; + private readonly IPullRequestBuilder _pullRequestBuilder; + // TODO (https://github.com/dotnet/arcade-services/issues/3866): When removed, remove the mocks from tests + private readonly IWorkItemProducerFactory _workItemProducerFactory; + private readonly ILogger _logger; + + protected readonly IReminderManager _pullRequestUpdateReminders; + protected readonly IReminderManager _pullRequestCheckReminders; + protected readonly IRedisCache _pullRequestState; + protected readonly IRedisCache _codeFlowState; /// /// Creates a new PullRequestActor /// public PullRequestActor( - PullRequestActorId id) + PullRequestActorId id, + IMergePolicyEvaluator mergePolicyEvaluator, + IRemoteFactory remoteFactory, + IActorFactory actorFactory, + ICoherencyUpdateResolver coherencyUpdateResolver, + IPullRequestBuilder pullRequestBuilder, + IRedisCacheFactory cacheFactory, + IReminderManagerFactory reminderManagerFactory, + IWorkItemProducerFactory workItemProducerFactory, + ILogger logger) { _id = id; + _mergePolicyEvaluator = mergePolicyEvaluator; + _remoteFactory = remoteFactory; + _actorFactory = actorFactory; + _coherencyUpdateResolver = coherencyUpdateResolver; + _pullRequestBuilder = pullRequestBuilder; + _workItemProducerFactory = workItemProducerFactory; + _logger = logger; + + _pullRequestUpdateReminders = reminderManagerFactory.CreateReminderManager(id.Id); + _pullRequestCheckReminders = reminderManagerFactory.CreateReminderManager(id.Id); + _pullRequestState = cacheFactory.Create(id.Id); + _codeFlowState = cacheFactory.Create(id.Id); } protected abstract Task<(string repository, string branch)> GetTargetAsync(); @@ -35,9 +80,57 @@ public PullRequestActor( /// /// True if updates have been applied; otherwise. /// - public Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem update) + public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem update) { - throw new NotImplementedException(); + _logger.LogInformation("Processing pending updates for subscription {subscriptionId}", update.SubscriptionId); + + // Check if we have an on-going PR for this actor + InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); + + if (pr == null) + { + _logger.LogInformation("No existing pull request state found"); + } + else + { + var canUpdate = await SynchronizeInProgressPullRequestAsync(pr); + if (!canUpdate) + { + _logger.LogInformation("PR {url} for subscription {subscriptionId} cannot be updated at this time", pr.Url, update.SubscriptionId); + await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDuration); + await _pullRequestCheckReminders.UnsetReminderAsync(); + return false; + } + } + + // Code flow updates are handled separetely + if (update.SubscriptionType == SubscriptionType.DependenciesAndSources) + { + return await ProcessCodeFlowUpdateAsync(update, pr); + } + + // If we have an existing PR, update it + if (pr != null) + { + await UpdatePullRequestAsync(pr, update); + _logger.LogInformation("Pull request {url} for subscription {subscriptionId} was updated", pr.Url, update.SubscriptionId); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + return true; + } + + // Create a new (regular) dependency update PR + var prUrl = await CreatePullRequestAsync(update); + if (prUrl == null) + { + _logger.LogInformation("No changes required for subscription {subscriptionId}, no pull request created", update.SubscriptionId); + } + else + { + _logger.LogInformation("Pull request '{url}' for subscription {subscriptionId} created", prUrl, update.SubscriptionId); + } + + await _pullRequestUpdateReminders.UnsetReminderAsync(); + return true; } protected virtual Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgressPullRequest pr) @@ -52,13 +145,234 @@ protected virtual Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgre /// This will evaluate merge policies on an in progress pull request and merge the pull request if policies allow. /// /// - /// A containing: - /// The current open pull request if one exists, and - /// if the open pull request can be updated; otherwise. + /// True, if the open pull request can be updated. /// - public virtual Task<(InProgressPullRequest? pr, bool canUpdate)> SynchronizeInProgressPullRequestAsync(InProgressPullRequest pullRequestCheck) + public virtual async Task SynchronizeInProgressPullRequestAsync(InProgressPullRequest pullRequestCheck) + { + if (string.IsNullOrEmpty(pullRequestCheck.Url)) + { + await _pullRequestState.TryDeleteAsync(); + await _codeFlowState.TryDeleteAsync(); + _logger.LogWarning("Removing invalid PR {url} from state memory", pullRequestCheck.Url); + return false; + } + + PullRequestStatus result = await GetPullRequestStateAsync(pullRequestCheck); + + _logger.LogInformation("Pull request {url} is {result}", pullRequestCheck.Url, result); + + switch (result) + { + // If the PR was merged or closed, we are done with it and the actor doesn't + // need to periodically run the synchronization any longer. + case PullRequestStatus.Completed: + case PullRequestStatus.UnknownPR: + return false; + case PullRequestStatus.InProgressCanUpdate: + await SetPullRequestCheckReminder(pullRequestCheck); + return true; + case PullRequestStatus.InProgressCannotUpdate: + await SetPullRequestCheckReminder(pullRequestCheck); + return false; + case PullRequestStatus.Invalid: + // We could have gotten here if there was an exception during + // the synchronization process. This was typical in the past + // when we would regularly get credential exceptions on github tokens + // that were just obtained. We don't want to unregister the reminder in these cases. + return false; + default: + _logger.LogError("Unknown pull request synchronization result {result}", result); + return false; + } + } + + private async Task GetPullRequestStateAsync(InProgressPullRequest pr) { - throw new NotImplementedException(); + var prUrl = pr.Url; + _logger.LogInformation("Synchronizing pull request {prUrl}", prUrl); + + (var targetRepository, _) = await GetTargetAsync(); + IRemote remote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + + _logger.LogInformation("Getting status for pull request: {url}", prUrl); + PrStatus status = await remote.GetPullRequestStatusAsync(prUrl); + _logger.LogInformation("Pull request {url} is {status}", prUrl, status); + switch (status) + { + // If the PR is currently open, then evaluate the merge policies, which will potentially + // merge the PR if they are successful. + case PrStatus.Open: + var mergePolicyResult = await CheckMergePolicyAsync(pr, remote); + + _logger.LogInformation("Policy check status for pull request {url} is {result}", prUrl, mergePolicyResult); + + switch (mergePolicyResult) + { + case MergePolicyCheckResult.Merged: + await UpdateSubscriptionsForMergedPRAsync(pr.ContainedSubscriptions); + await AddDependencyFlowEventsAsync( + pr.ContainedSubscriptions, + DependencyFlowEventType.Completed, + DependencyFlowEventReason.AutomaticallyMerged, + mergePolicyResult, + prUrl); + + await _pullRequestState.TryDeleteAsync(); + await _codeFlowState.TryDeleteAsync(); + return PullRequestStatus.Completed; + + case MergePolicyCheckResult.FailedPolicies: + await TagSourceRepositoryGitHubContactsIfPossibleAsync(pr); + goto case MergePolicyCheckResult.FailedToMerge; + + case MergePolicyCheckResult.NoPolicies: + case MergePolicyCheckResult.FailedToMerge: + return PullRequestStatus.InProgressCanUpdate; + + case MergePolicyCheckResult.PendingPolicies: + return PullRequestStatus.InProgressCannotUpdate; + + default: + throw new NotImplementedException($"Unknown merge policy check result {mergePolicyResult}"); + } + + case PrStatus.Merged: + case PrStatus.Closed: + // If the PR has been merged, update the subscription information + if (status == PrStatus.Merged) + { + await UpdateSubscriptionsForMergedPRAsync(pr.ContainedSubscriptions); + } + + DependencyFlowEventReason reason = status == PrStatus.Merged + ? DependencyFlowEventReason.ManuallyMerged + : DependencyFlowEventReason.ManuallyClosed; + + await AddDependencyFlowEventsAsync( + pr.ContainedSubscriptions, + DependencyFlowEventType.Completed, + reason, + pr.MergePolicyResult, + prUrl); + + await _pullRequestState.TryDeleteAsync(); + await _codeFlowState.TryDeleteAsync(); + + // Also try to clean up the PR branch. + try + { + _logger.LogInformation("Trying to clean up the branch for pull request {url}", prUrl); + await remote.DeletePullRequestBranchAsync(prUrl); + } + catch (DarcException) + { + _logger.LogInformation("Failed to delete branch associated with pull request {url}", prUrl); + } + + _logger.LogInformation("PR has been manually {action}", status); + return PullRequestStatus.Completed; + + default: + throw new NotImplementedException($"Unknown PR status '{status}'"); + } + } + + /// + /// Check the merge policies for a PR and merge if they have succeeded. + /// + /// Pull request + /// Darc remote + /// Result of the policy check. + private async Task CheckMergePolicyAsync(InProgressPullRequest pr, IRemote remote) + { + IReadOnlyList policyDefinitions = await GetMergePolicyDefinitions(); + MergePolicyEvaluationResults result = await _mergePolicyEvaluator.EvaluateAsync(pr, remote, policyDefinitions); + + await UpdateMergeStatusAsync(remote, pr.Url, result.Results); + + // As soon as one policy is actively failed, we enter a failed state. + if (result.Failed) + { + _logger.LogInformation("NOT Merged: PR '{url}' failed policies {policies}", + pr.Url, + string.Join(", ", result.Results.Where(r => r.Status != MergePolicyEvaluationStatus.Success).Select(r => r.MergePolicyInfo.Name + r.Title))); + + return MergePolicyCheckResult.FailedPolicies; + } + + if (result.Pending) + { + _logger.LogInformation("NOT Merged: PR '{url}' has pending policies {policies}", + pr.Url, + string.Join(", ", result.Results.Where(r => r.Status == MergePolicyEvaluationStatus.Pending).Select(r => r.MergePolicyInfo.Name + r.Title))); + return MergePolicyCheckResult.PendingPolicies; + } + + if (!result.Succeeded) + { + _logger.LogInformation("NOT Merged: PR '{url}' There are no merge policies", pr.Url); + return MergePolicyCheckResult.NoPolicies; + } + + try + { + await remote.MergeDependencyPullRequestAsync(pr.Url, new MergePullRequestParameters()); + } + catch + { + _logger.LogInformation("NOT Merged: PR '{url}' has merge conflicts", pr.Url); + return MergePolicyCheckResult.FailedToMerge; + } + + var passedPolicies = string.Join(", ", policyDefinitions.Select(p => p.Name)); + _logger.LogInformation("Merged: PR '{url}' passed policies {passedPolicies}", pr.Url, passedPolicies); + return MergePolicyCheckResult.Merged; + } + + /// + /// Create new checks or update the status of existing checks for a PR. + /// + /// Pull request URL + /// Darc remote + /// List of merge policies + /// Result of the policy check. + private static Task UpdateMergeStatusAsync(IRemote remote, string prUrl, IReadOnlyList evaluations) + { + return remote.CreateOrUpdatePullRequestMergeStatusInfoAsync(prUrl, evaluations); + } + + private async Task UpdateSubscriptionsForMergedPRAsync(IEnumerable subscriptionPullRequestUpdates) + { + _logger.LogInformation("Updating subscriptions for merged PR"); + foreach (SubscriptionPullRequestUpdate update in subscriptionPullRequestUpdates) + { + ISubscriptionActor actor = _actorFactory.CreateSubscriptionActor(update.SubscriptionId); + if (!await actor.UpdateForMergedPullRequestAsync(update.BuildId)) + { + _logger.LogInformation("Failed to update subscription {subscriptionId} for merged PR", update.SubscriptionId); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + await _pullRequestCheckReminders.UnsetReminderAsync(); + await _pullRequestState.TryDeleteAsync(); + await _codeFlowState.TryDeleteAsync(); + } + } + } + + private async Task AddDependencyFlowEventsAsync( + IEnumerable subscriptionPullRequestUpdates, + DependencyFlowEventType flowEvent, + DependencyFlowEventReason reason, + MergePolicyCheckResult policy, + string? prUrl) + { + foreach (SubscriptionPullRequestUpdate update in subscriptionPullRequestUpdates) + { + ISubscriptionActor actor = _actorFactory.CreateSubscriptionActor(update.SubscriptionId); + if (!await actor.AddDependencyFlowEventAsync(update.BuildId, flowEvent, reason, policy, "PR", prUrl)) + { + _logger.LogInformation("Failed to add dependency flow event for {subscriptionId}", update.SubscriptionId); + } + } } /// @@ -78,7 +392,7 @@ protected virtual Task TagSourceRepositoryGitHubContactsIfPossibleAsync(InProgre /// PRs are marked as non-updateable so that we can allow pull request checks to complete on a PR prior /// to pushing additional commits. /// - public Task UpdateAssetsAsync( + public async Task UpdateAssetsAsync( Guid subscriptionId, SubscriptionType type, int buildId, @@ -86,6 +400,627 @@ public Task UpdateAssetsAsync( string sourceSha, List assets) { - throw new NotImplementedException(); + // Check if we have an on-going PR for this actor + InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); + bool canUpdate; + if (pr == null) + { + _logger.LogInformation("No existing pull request state found"); + canUpdate = true; + } + else + { + canUpdate = await SynchronizeInProgressPullRequestAsync(pr); + } + + var update = new SubscriptionUpdateWorkItem + { + ActorId = _id.ToString(), + SubscriptionId = subscriptionId, + SubscriptionType = type, + BuildId = buildId, + SourceSha = sourceSha, + SourceRepo = sourceRepo, + Assets = assets, + IsCoherencyUpdate = false, + }; + + // Regardless of code flow or regular PR, if the PR are not complete, postpone the update + if (pr != null && !canUpdate) + { + await _pullRequestUpdateReminders.SetReminderAsync(update, DefaultReminderDuration); + _logger.LogInformation("Pull request '{prUrl}' cannot be updated, update queued", pr!.Url); + return true; + } + + if (type == SubscriptionType.DependenciesAndSources) + { + return await ProcessCodeFlowUpdateAsync(update, pr); + } + + try + { + if (pr == null) + { + var prUrl = await CreatePullRequestAsync(update); + if (prUrl == null) + { + _logger.LogInformation("Updates require no changes, no pull request created"); + } + else + { + _logger.LogInformation("Pull request '{prUrl}' created", prUrl); + } + + return true; + } + + await UpdatePullRequestAsync(pr, update); + } + catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int)HttpStatusCode.Unauthorized).ToString())) + { + // We want to preserve the HttpRequestException's information but it's not serializable + // We'll log the full exception object so it's in Application Insights, and strip any single quotes from the message to ensure + // GitHub issues are properly created. + _logger.LogError(reqEx, "Failure to authenticate to repository"); + return false; + } + + return true; + } + + /// + /// Creates a pull request from the given updates. + /// + /// The pull request url when a pr was created; if no PR is necessary + private async Task CreatePullRequestAsync(SubscriptionUpdateWorkItem update) + { + (var targetRepository, var targetBranch) = await GetTargetAsync(); + + IRemote darcRemote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + + TargetRepoDependencyUpdate repoDependencyUpdate = + await GetRequiredUpdates(update, _remoteFactory, targetRepository, prBranch: null, targetBranch); + + if (repoDependencyUpdate.CoherencyCheckSuccessful && repoDependencyUpdate.RequiredUpdates.Count < 1) + { + return null; + } + + var newBranchName = GetNewBranchName(targetBranch); + await darcRemote.CreateNewBranchAsync(targetRepository, targetBranch, newBranchName); + + try + { + var description = await _pullRequestBuilder.CalculatePRDescriptionAndCommitUpdatesAsync( + repoDependencyUpdate.RequiredUpdates, + currentDescription: null, + targetRepository, + newBranchName); + + var inProgressPr = new InProgressPullRequest + { + ActorId = _id.ToString(), + + // Calculate the subscriptions contained within the + // update. Coherency updates do not have subscription info. + ContainedSubscriptions = repoDependencyUpdate.RequiredUpdates + .Where(u => !u.update.IsCoherencyUpdate) + .Select( + u => new SubscriptionPullRequestUpdate + { + SubscriptionId = u.update.SubscriptionId, + BuildId = u.update.BuildId + }) + .ToList(), + + RequiredUpdates = repoDependencyUpdate.RequiredUpdates + .SelectMany(update => update.deps) + .Select(du => new DependencyUpdateSummary + { + DependencyName = du.To.Name, + FromVersion = du.From.Version, + ToVersion = du.To.Version + }) + .ToList(), + + CoherencyCheckSuccessful = repoDependencyUpdate.CoherencyCheckSuccessful, + CoherencyErrors = repoDependencyUpdate.CoherencyErrors + }; + + var prUrl = await darcRemote.CreatePullRequestAsync( + targetRepository, + new PullRequest + { + Title = await _pullRequestBuilder.GeneratePRTitleAsync(inProgressPr, targetBranch), + Description = description, + BaseBranch = targetBranch, + HeadBranch = newBranchName, + }); + + if (!string.IsNullOrEmpty(prUrl)) + { + inProgressPr.Url = prUrl; + + await AddDependencyFlowEventsAsync( + inProgressPr.ContainedSubscriptions, + DependencyFlowEventType.Created, + DependencyFlowEventReason.New, + MergePolicyCheckResult.PendingPolicies, + prUrl); + + await SetPullRequestCheckReminder(inProgressPr); + return prUrl; + } + + // If we did not create a PR, then mark the dependency flow as completed as nothing to do. + await AddDependencyFlowEventsAsync( + inProgressPr.ContainedSubscriptions, + DependencyFlowEventType.Completed, + DependencyFlowEventReason.NothingToDo, + MergePolicyCheckResult.PendingPolicies, + null); + + // Something wrong happened when trying to create the PR but didn't throw an exception (probably there was no diff). + // We need to delete the branch also in this case. + await darcRemote.DeleteBranchAsync(targetRepository, newBranchName); + return null; + } + catch + { + await darcRemote.DeleteBranchAsync(targetRepository, newBranchName); + throw; + } } + + private async Task UpdatePullRequestAsync(InProgressPullRequest pr, SubscriptionUpdateWorkItem update) + { + (var targetRepository, var targetBranch) = await GetTargetAsync(); + + _logger.LogInformation("Updating pull request {url} branch {targetBranch} in {targetRepository}", pr.Url, targetBranch, targetRepository); + + IRemote darcRemote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + PullRequest pullRequest = await darcRemote.GetPullRequestAsync(pr.Url); + + TargetRepoDependencyUpdate targetRepositoryUpdates = + await GetRequiredUpdates(update, _remoteFactory, targetRepository, pullRequest.HeadBranch, targetBranch); + + if (targetRepositoryUpdates.CoherencyCheckSuccessful && targetRepositoryUpdates.RequiredUpdates.Count < 1) + { + _logger.LogInformation("No updates found for pull request {url}", pr.Url); + return; + } + + _logger.LogInformation("Found {count} required updates for pull request {url}", targetRepositoryUpdates.RequiredUpdates.Count, pr.Url); + + pr.RequiredUpdates = MergeExistingWithIncomingUpdates(pr.RequiredUpdates, targetRepositoryUpdates.RequiredUpdates); + + if (pr.RequiredUpdates.Count < 1) + { + _logger.LogInformation("No new updates found for pull request {url}", pr.Url); + return; + } + + pr.CoherencyCheckSuccessful = targetRepositoryUpdates.CoherencyCheckSuccessful; + pr.CoherencyErrors = targetRepositoryUpdates.CoherencyErrors; + + List previousSubscriptions = [.. pr.ContainedSubscriptions]; + + // Update the list of contained subscriptions with the new subscription update. + // Replace all existing updates for the subscription id with the new update. + // This avoids a potential issue where we may update the last applied build id + // on the subscription to an older build id. + pr.ContainedSubscriptions.RemoveAll(s => s.SubscriptionId == update.SubscriptionId); + + // Mark all previous dependency updates that are being updated as Updated. All new dependencies should not be + // marked as update as they are new. Any dependency not being updated should not be marked as failed. + // At this point, pr.ContainedSubscriptions only contains the subscriptions that were not updated, + // so everything that is in the previous list but not in the current list were updated. + await AddDependencyFlowEventsAsync( + previousSubscriptions.Except(pr.ContainedSubscriptions), + DependencyFlowEventType.Updated, + DependencyFlowEventReason.FailedUpdate, + pr.MergePolicyResult, + pr.Url); + + pr.ContainedSubscriptions.AddRange(targetRepositoryUpdates.RequiredUpdates + .Where(u => !u.update.IsCoherencyUpdate) + .Select( + u => new SubscriptionPullRequestUpdate + { + SubscriptionId = u.update.SubscriptionId, + BuildId = u.update.BuildId + })); + + // Mark any new dependency updates as Created. Any subscriptions that are in pr.ContainedSubscriptions + // but were not in the previous list of subscriptions are new + await AddDependencyFlowEventsAsync( + pr.ContainedSubscriptions.Except(previousSubscriptions), + DependencyFlowEventType.Created, + DependencyFlowEventReason.New, + MergePolicyCheckResult.PendingPolicies, + pr.Url); + + pullRequest.Description = await _pullRequestBuilder.CalculatePRDescriptionAndCommitUpdatesAsync( + targetRepositoryUpdates.RequiredUpdates, + pullRequest.Description, + targetRepository, + pullRequest.HeadBranch); + + pullRequest.Title = await _pullRequestBuilder.GeneratePRTitleAsync(pr, targetBranch); + + await darcRemote.UpdatePullRequestAsync(pr.Url, pullRequest); + await SetPullRequestCheckReminder(pr); + + _logger.LogInformation("Pull request '{prUrl}' updated", pr.Url); + } + + /// + /// Merges the list of existing updates in a PR with a list of incoming updates + /// + /// pr object to update + /// list of new incoming updates + /// Merged list of existing updates along with the new + private static List MergeExistingWithIncomingUpdates( + List existingUpdates, + List<(SubscriptionUpdateWorkItem update, List deps)> incomingUpdates) + { + // First project the new updates to the final list + var mergedUpdates = + incomingUpdates.SelectMany(update => update.deps) + .Select(du => new DependencyUpdateSummary + { + DependencyName = du.To.Name, + FromVersion = du.From.Version, + ToVersion = du.To.Version + }).ToList(); + + // Project to a form that is easy to search + var searchableUpdates = + mergedUpdates.Select(u => u.DependencyName).ToHashSet(StringComparer.OrdinalIgnoreCase); + + // Add any existing assets that weren't modified by the incoming update + if (existingUpdates != null) + { + foreach (DependencyUpdateSummary update in existingUpdates) + { + if (!searchableUpdates.Contains(update.DependencyName)) + { + mergedUpdates.Add(update); + } + } + } + + return mergedUpdates; + } + + private class TargetRepoDependencyUpdate + { + public bool CoherencyCheckSuccessful { get; set; } = true; + public List? CoherencyErrors { get; set; } + public List<(SubscriptionUpdateWorkItem update, List deps)> RequiredUpdates { get; set; } = []; + } + + /// + /// Given a set of input updates from builds, determine what updates + /// are required in the target repository. + /// + /// Updates + /// Target repository to calculate updates for + /// PR head branch + /// Target branch + /// Darc remote factory + /// List of updates and dependencies that need updates. + /// + /// This is done in two passes. The first pass runs through and determines the non-coherency + /// updates required based on the input updates. The second pass uses the repo state + the + /// updates from the first pass to determine what else needs to change based on the coherency metadata. + /// + private async Task GetRequiredUpdates( + SubscriptionUpdateWorkItem update, + IRemoteFactory remoteFactory, + string targetRepository, + string? prBranch, + string targetBranch) + { + _logger.LogInformation("Getting Required Updates for {branch} of {targetRepository}", targetBranch, targetRepository); + // Get a remote factory for the target repo + IRemote darc = await remoteFactory.GetRemoteAsync(targetRepository, _logger); + + TargetRepoDependencyUpdate repoDependencyUpdate = new(); + + // Existing details + var existingDependencies = (await darc.GetDependenciesAsync(targetRepository, prBranch ?? targetBranch)).ToList(); + + IEnumerable assetData = update.Assets.Select( + a => new AssetData(false) + { + Name = a.Name, + Version = a.Version + }); + + // Retrieve the source of the assets + List dependenciesToUpdate = _coherencyUpdateResolver.GetRequiredNonCoherencyUpdates( + update.SourceRepo, + update.SourceSha, + assetData, + existingDependencies); + + if (dependenciesToUpdate.Count < 1) + { + // No dependencies need to be updated. + await UpdateSubscriptionsForMergedPRAsync( + new List + { + new() + { + SubscriptionId = update.SubscriptionId, + BuildId = update.BuildId + } + }); + } + else + { + // Update the existing details list + foreach (DependencyUpdate dependencyUpdate in dependenciesToUpdate) + { + existingDependencies.Remove(dependencyUpdate.From); + existingDependencies.Add(dependencyUpdate.To); + } + + repoDependencyUpdate.RequiredUpdates.Add((update, dependenciesToUpdate)); + } + + // Once we have applied all of non coherent updates, then we need to run a coherency check on the dependencies. + List coherencyUpdates = []; + try + { + _logger.LogInformation("Running a coherency check on the existing dependencies for branch {branch} of repo {repository}", + targetBranch, + targetRepository); + coherencyUpdates = await _coherencyUpdateResolver.GetRequiredCoherencyUpdatesAsync(existingDependencies, remoteFactory); + } + catch (DarcCoherencyException e) + { + _logger.LogInformation("Failed attempting strict coherency update on branch '{strictCoherencyFailedBranch}' of repo '{strictCoherencyFailedRepo}'", + targetBranch, targetRepository); + repoDependencyUpdate.CoherencyCheckSuccessful = false; + repoDependencyUpdate.CoherencyErrors = e.Errors.Select(e => new CoherencyErrorDetails + { + Error = e.Error, + PotentialSolutions = e.PotentialSolutions + }).ToList(); + } + + if (coherencyUpdates.Count != 0) + { + // For the update asset parameters, we don't have any information on the source of the update, + // since coherency can be run even without any updates. + var coherencyUpdateParameters = new SubscriptionUpdateWorkItem + { + ActorId = _id.Id, + IsCoherencyUpdate = true + }; + repoDependencyUpdate.RequiredUpdates.Add((coherencyUpdateParameters, coherencyUpdates.ToList())); + } + + _logger.LogInformation("Finished getting Required Updates for {branch} of {targetRepository}", targetBranch, targetRepository); + return repoDependencyUpdate; + } + + private static string GetNewBranchName(string targetBranch) => $"darc-{targetBranch}-{Guid.NewGuid()}"; + + private async Task SetPullRequestCheckReminder(InProgressPullRequest prState) + { + await _pullRequestCheckReminders.SetReminderAsync(prState, DefaultReminderDuration); + await _pullRequestState.SetAsync(prState); + } + + #region Code flow subscriptions + + /// + /// Alternative to ProcessPendingUpdatesAsync that is used in the code flow (VMR) scenario. + /// + private async Task ProcessCodeFlowUpdateAsync( + SubscriptionUpdateWorkItem update, + InProgressPullRequest? pr) + { + CodeFlowStatus? codeFlowStatus = await _codeFlowState.TryGetStateAsync(); + + // The E2E order of things for is: + // 1. We send a request to PCS and wait for a branch to be created. We note down this in the codeflow status. We set a reminder. + // 2. When reminder kicks in, we check if the branch is created. If not, we repeat the reminder. + // 3. When branch is created, we create the PR and set the usual reminder of watching a PR (common with the regular subscriptions). + // 4. For new updates, we only delegate those to PCS which will push in the branch. + if (pr == null) + { + (var targetRepository, var targetBranch) = await GetTargetAsync(); + + // Step 1. Let PCS create a branch for us + if (codeFlowStatus == null) + { + return await RequestCodeFlowBranchAsync(update, targetBranch); + } + + // Step 2. Wait for the branch to be created + IRemote remote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + if (!await remote.BranchExistsAsync(targetRepository, codeFlowStatus.PrBranch)) + { + _logger.LogInformation("Branch {branch} for subscription {subscriptionId} not created yet. Will check again later", + codeFlowStatus.PrBranch, + update.SubscriptionId); + + await _pullRequestUpdateReminders.SetReminderAsync(update, TimeSpan.FromMinutes(3)); + return true; + } + + // Step 3. Create a PR + var prUrl = await CreateCodeFlowPullRequestAsync( + update, + targetRepository, + codeFlowStatus.PrBranch, + targetBranch); + + _logger.LogInformation("Pending updates applied. PR {prUrl} created", prUrl); + return true; + } + + // Technically, this should never happen as we create the code flow data before we even create the PR + if (codeFlowStatus == null) + { + _logger.LogError("Missing code flow data for subscription {subscription}", update.SubscriptionId); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + await _pullRequestCheckReminders.UnsetReminderAsync(); + return false; + } + + // Step 4. Update the PR (if needed) + + // Compare last SHA with the build SHA to see if we need to delegate this update to PCS + if (update.SourceSha == codeFlowStatus.SourceSha) + { + _logger.LogInformation("PR {url} for {subscription} is up to date ({sha})", + pr.Url, + update.SubscriptionId, + update.SourceSha); + return false; + } + + try + { + // TODO (https://github.com/dotnet/arcade-services/issues/3866): Execute code flow logic immediately + var producer = _workItemProducerFactory.CreateProducer(); + await producer.ProduceWorkItemAsync(new CodeFlowWorkItem + { + BuildId = update.BuildId, + SubscriptionId = update.SubscriptionId, + PrBranch = codeFlowStatus.PrBranch, + PrUrl = pr.Url, + }); + + codeFlowStatus.SourceSha = update.SourceSha; + + // TODO (https://github.com/dotnet/arcade-services/issues/3866): We need to update the InProgressPullRequest fully, assets and other info just like we do in UpdatePullRequestAsync + // Right now, we are not flowing packages in codeflow subscriptions yet, so this functionality is no there + // For now, we manually update the info the unit tests expect + pr.ContainedSubscriptions.Clear(); + pr.ContainedSubscriptions.Add(new SubscriptionPullRequestUpdate + { + SubscriptionId = update.SubscriptionId, + BuildId = update.BuildId + }); + + await _codeFlowState.SetAsync(codeFlowStatus); + await SetPullRequestCheckReminder(pr); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + } + catch (Exception e) + { + // TODO https://github.com/dotnet/arcade-services/issues/3318: Handle this + _logger.LogError(e, "Failed to request branch update for PR {url} for subscription {subscriptionId}", + pr.Url, + update.SubscriptionId); + } + + _logger.LogInformation("New code flow changes requested"); + return true; + } + + private async Task RequestCodeFlowBranchAsync(SubscriptionUpdateWorkItem update, string targetBranch) + { + CodeFlowStatus codeFlowUpdate = new() + { + PrBranch = GetNewBranchName(targetBranch), + SourceSha = update.SourceSha, + }; + + _logger.LogInformation( + "New code flow request for subscription {subscriptionId}. Requesting branch {branch} from PCS", + update.SubscriptionId, + codeFlowUpdate.PrBranch); + + try + { + // TODO (https://github.com/dotnet/arcade-services/issues/3866): Execute code flow logic immediately + var producer = _workItemProducerFactory.CreateProducer(); + await producer.ProduceWorkItemAsync(new CodeFlowWorkItem + { + BuildId = update.BuildId, + SubscriptionId = update.SubscriptionId, + PrBranch = codeFlowUpdate.PrBranch, + }); + } + catch (Exception e) + { + // TODO https://github.com/dotnet/arcade-services/issues/3318: Handle this + _logger.LogError(e, "Failed to request new branch {branch} for subscription {subscriptionId}", + codeFlowUpdate.PrBranch, + update.SubscriptionId); + return false; + } + + await _codeFlowState.SetAsync(codeFlowUpdate); + await _pullRequestUpdateReminders.SetReminderAsync(update, TimeSpan.FromMinutes(3)); + + _logger.LogInformation("Pending updates applied. Branch {prBranch} requested from PCS", codeFlowUpdate.PrBranch); + return true; + } + + private async Task CreateCodeFlowPullRequestAsync( + SubscriptionUpdateWorkItem update, + string targetRepository, + string prBranch, + string targetBranch) + { + IRemote darcRemote = await _remoteFactory.GetRemoteAsync(targetRepository, _logger); + + try + { + var title = await _pullRequestBuilder.GenerateCodeFlowPRTitleAsync(update, targetBranch); + var description = await _pullRequestBuilder.GenerateCodeFlowPRDescriptionAsync(update); + + var prUrl = await darcRemote.CreatePullRequestAsync( + targetRepository, + new PullRequest + { + Title = title, + Description = description, + BaseBranch = targetBranch, + HeadBranch = prBranch, + }); + + InProgressPullRequest inProgressPr = new() + { + ActorId = _id.ToString(), + Url = prUrl, + ContainedSubscriptions = + [ + new() + { + SubscriptionId = update.SubscriptionId, + BuildId = update.BuildId + } + ], + }; + + await AddDependencyFlowEventsAsync( + inProgressPr.ContainedSubscriptions, + DependencyFlowEventType.Created, + DependencyFlowEventReason.New, + MergePolicyCheckResult.PendingPolicies, + prUrl); + + await SetPullRequestCheckReminder(inProgressPr); + await _pullRequestUpdateReminders.UnsetReminderAsync(); + + return prUrl; + } + catch + { + await darcRemote.DeleteBranchAsync(targetRepository, prBranch); + throw; + } + } + + #endregion } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs index d47d2dd24b..f4cbac0e15 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/ActorWorkItem.cs @@ -11,5 +11,5 @@ namespace ProductConstructionService.DependencyFlow.WorkItems; public abstract class ActorWorkItem : WorkItem { [DataMember] - public string ActorId { get; set; } + public required string ActorId { get; init; } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs index a9bc17dc2b..2efb2f1d0f 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/InProgressPullRequest.cs @@ -20,13 +20,13 @@ public class InProgressPullRequest : ActorWorkItem, IPullRequest public List CoherencyErrors { get; set; } [DataMember] - public MergePolicyCheckResult MergePolicyResult { get; set; } + public MergePolicyCheckResult MergePolicyResult { get; init; } [DataMember] - public List ContainedSubscriptions { get; set; } + public List ContainedSubscriptions { get; init; } [DataMember] - public List Contained { get; set; } + public List Contained { get; init; } [DataMember] public List RequiredUpdates { get; set; } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs index f909a5cc55..5e51ac2807 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItems/SubscriptionUpdateWorkItem.cs @@ -11,27 +11,27 @@ namespace ProductConstructionService.DependencyFlow.WorkItems; public class SubscriptionUpdateWorkItem : ActorWorkItem { [DataMember] - public Guid SubscriptionId { get; set; } + public Guid SubscriptionId { get; init; } [DataMember] - public SubscriptionType SubscriptionType { get; set; } + public SubscriptionType SubscriptionType { get; init; } [DataMember] - public int BuildId { get; set; } + public int BuildId { get; init; } [DataMember] - public string SourceSha { get; set; } + public string SourceSha { get; init; } [DataMember] - public string SourceRepo { get; set; } + public string SourceRepo { get; init; } [DataMember] - public List Assets { get; set; } + public List Assets { get; init; } /// /// If true, this is a coherency update and not driven by specific /// subscription ids (e.g. could be multiple if driven by a batched subscription) /// [DataMember] - public bool IsCoherencyUpdate { get; set; } + public bool IsCoherencyUpdate { get; init; } } diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index bed8c61b99..6878fa24c0 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -31,7 +31,7 @@ public SubscriptionTriggerer( public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequency) { - var workItemProducer = _workItemProducerFactory.CreateProducer(); + var workItemProducer = _workItemProducerFactory.CreateProducer(); foreach (var updateSubscriptionWorkItem in await GetSubscriptionsToTrigger(targetUpdateFrequency)) { await workItemProducer.ProduceWorkItemAsync(updateSubscriptionWorkItem); @@ -41,9 +41,9 @@ public async Task TriggerSubscriptionsAsync(UpdateFrequency targetUpdateFrequenc } } - private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) + private async Task> GetSubscriptionsToTrigger(UpdateFrequency targetUpdateFrequency) { - List subscriptionsToTrigger = new(); + List subscriptionsToTrigger = new(); var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) @@ -79,10 +79,10 @@ private async Task> GetSubscriptionsToTrigger(U if (isThereAnUnappliedBuildInTargetChannel && latestBuildInTargetChannel != null) { - subscriptionsToTrigger.Add(new SubscriptionUpdateWorkItem + subscriptionsToTrigger.Add(new SubscriptionTriggerWorkItem { BuildId = latestBuildInTargetChannel.Id, - SubscriptionId = subscription.Id + SubscriptionId = subscription.Id, }); } } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs index 2d40877d0b..60d1bc1846 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItem.cs @@ -5,6 +5,6 @@ namespace ProductConstructionService.WorkItems; public abstract class WorkItem { - public Guid Id { get; set; } = Guid.NewGuid(); + public Guid Id { get; init; } = Guid.NewGuid(); public string Type => GetType().Name; } diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs index 84946bc258..ca4485c4ba 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20180716Tests.cs @@ -17,6 +17,8 @@ using Microsoft.Extensions.Logging; using Moq; using ProductConstructionService.Api.Api.v2018_07_16.Controllers; +using ProductConstructionService.DependencyFlow.WorkItems; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -163,6 +165,15 @@ public static async Task Dependencies(IServiceCollection collection) collection.AddSingleton(); collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); + collection.AddSingleton(new SubscriptionIdGenerator(RunningService.PCS)); + + var mockWorkItemProducerFactory = new Mock(); + var mockWorkItemProducer = new Mock>(); + mockWorkItemProducerFactory + .Setup(f => f.CreateProducer()) + .Returns(mockWorkItemProducer.Object); + + collection.AddSingleton(mockWorkItemProducerFactory.Object); } public static Func Clock(IServiceCollection collection) diff --git a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs index 5ced4553ff..2d412c6c06 100644 --- a/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs +++ b/test/ProductConstructionService.Api.Tests/ChannelsController20200220Tests.cs @@ -170,10 +170,13 @@ public static async Task Dependencies(IServiceCollection collection) }); collection.AddSingleton(Mock.Of()); collection.AddSingleton(Mock.Of()); + collection.AddSingleton(new SubscriptionIdGenerator(RunningService.PCS)); var mockWorkItemProducerFactory = new Mock(); var mockWorkItemProducer = new Mock>(); - mockWorkItemProducerFactory.Setup(f => f.CreateProducer()).Returns(mockWorkItemProducer.Object); + mockWorkItemProducerFactory + .Setup(f => f.CreateProducer()) + .Returns(mockWorkItemProducer.Object); collection.AddSingleton(mockWorkItemProducerFactory.Object); } diff --git a/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs index 289bad6799..7c3a3cdb72 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs @@ -96,18 +96,10 @@ public void ActorTests_SetUp() [TearDown] public void ActorTests_TearDown() { - Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders, options => options.ExcludingProperties()); - RedisCache.Data.Should().BeEquivalentTo(ExpectedActorState); - } - - protected void SetReminder(Subscription subscription, T reminder) where T : WorkItem - { - Reminders.Reminders[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = reminder; - } - - protected void RemoveReminder(Subscription subscription) where T : WorkItem - { - Reminders.Reminders.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + var ExcludeWorkItemId = static (FluentAssertions.Equivalency.EquivalencyAssertionOptions> opt) + => opt.Excluding(member => member.DeclaringType == typeof(WorkItem) && member.Name.Equals(nameof(WorkItem.Id))); + RedisCache.Data.Should().BeEquivalentTo(ExpectedActorState, ExcludeWorkItemId); + Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders, ExcludeWorkItemId); } protected void SetState(Subscription subscription, T state) where T : class diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs new file mode 100644 index 0000000000..8fade6df40 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs @@ -0,0 +1,88 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Maestro.Data.Models; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture, NonParallelizable] +internal class PendingCodeFlowUpdatesTests : PendingUpdatePullRequestActorTests +{ + [Test] + public async Task PendingCodeFlowUpdatesNotUpdatablePr() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build build = GivenANewBuild(true); + + WithExistingCodeFlowStatus(build); + WithExistingPrBranch(); + AndPendingUpdates(build, isCodeFlow: true); + + WithExistingCodeFlowPullRequest(build, canUpdate: false); + await WhenProcessPendingUpdatesAsyncIsCalled(build, isCodeFlow: true); + + ThenPcsShouldNotHaveBeenCalled(build, InProgressPrUrl); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + AndShouldHaveInProgressPullRequestState(build); + } + + [Test] + public async Task PendingUpdatesUpdatablePrButNoNewBuild() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build build = GivenANewBuild(true); + + WithExistingCodeFlowStatus(build); + WithExistingPrBranch(); + WithExistingCodeFlowPullRequest(build, canUpdate: true); + + await WhenProcessPendingUpdatesAsyncIsCalled(build, isCodeFlow: true); + + ThenPcsShouldNotHaveBeenCalled(build, InProgressPrUrl); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + AndShouldHaveInProgressPullRequestState(build); + AndShouldHavePullRequestCheckReminder(build); + } + + [Test] + public async Task PendingUpdatesUpdatablePr() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build oldBuild = GivenANewBuild(true); + Build newBuild = GivenANewBuild(true); + newBuild.Commit = "sha456"; + + WithExistingCodeFlowStatus(oldBuild); + WithExistingPrBranch(); + + WithExistingCodeFlowPullRequest(oldBuild, canUpdate: true); + await WhenProcessPendingUpdatesAsyncIsCalled(newBuild, isCodeFlow: true); + + ThenPcsShouldHaveBeenCalled(newBuild, InProgressPrUrl, out var prBranch); + AndShouldHaveNoPendingUpdateState(); + AndShouldHavePullRequestCheckReminder(newBuild); + prBranch.Should().Be(InProgressPrHeadBranch); + AndShouldHaveCodeFlowState(newBuild, InProgressPrHeadBranch); + AndShouldHaveInProgressPullRequestState(newBuild); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs new file mode 100644 index 0000000000..bd19a725dd --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; +using ProductConstructionService.DependencyFlow.WorkItems; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal abstract class PendingUpdatePullRequestActorTests : PullRequestActorTests +{ + protected async Task WhenProcessPendingUpdatesAsyncIsCalled(Build forBuild, bool isCodeFlow = false) + { + await Execute( + async context => + { + IPullRequestActor actor = CreatePullRequestActor(context); + await actor.ProcessPendingUpdatesAsync(CreateSubscriptionUpdate(forBuild, isCodeFlow)); + }); + } + + protected void GivenAPendingUpdateReminder(Build forBuild, bool isCodeFlow = false) + { + SetExpectedReminder(Subscription, CreateSubscriptionUpdate(forBuild, isCodeFlow)); + } + + protected void AndNoPendingUpdates() + { + RemoveExpectedState(Subscription); + RemoveState(Subscription); + } + + protected void AndPendingUpdates(Build forBuild, bool isCodeFlow = false) + { + AfterDbUpdateActions.Add( + () => + { + var update = CreateSubscriptionUpdate(forBuild, isCodeFlow); + SetExpectedReminder(Subscription, update); + }); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs new file mode 100644 index 0000000000..fcdcacc735 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture, NonParallelizable] +internal class PendingUpdatesTests : PendingUpdatePullRequestActorTests +{ + [Test] + public async Task PendingUpdatesNotUpdatablePr() + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + GivenAPendingUpdateReminder(b); + AndPendingUpdates(b); + WithExistingPullRequest(b, canUpdate: false); + await WhenProcessPendingUpdatesAsyncIsCalled(b); + } + + [Test] + public async Task PendingUpdatesUpdatablePr() + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = true, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + GivenAPendingUpdateReminder(b); + AndPendingUpdates(b); + WithRequireNonCoherencyUpdates(); + WithNoRequiredCoherencyUpdates(); + WithExistingPullRequest(b, canUpdate: true); + + await WhenProcessPendingUpdatesAsyncIsCalled(b); + + ThenGetRequiredUpdatesShouldHaveBeenCalled(b, true); + ThenUpdateReminderIsRemoved(); + AndPendingUpdateIsRemoved(); + AndCommitUpdatesShouldHaveBeenCalled(b); + AndUpdatePullRequestShouldHaveBeenCalled(); + AndShouldHavePullRequestCheckReminder(b); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs new file mode 100644 index 0000000000..0115a3b32f --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs @@ -0,0 +1,472 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Maestro.Contracts; +using Maestro.Data; +using Maestro.Data.Models; +using Maestro.DataProviders; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.GitHub.Authentication; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.Services.Common; +using Moq; +using ProductConstructionService.DependencyFlow.WorkItems; +using Asset = Maestro.Contracts.Asset; +using AssetData = Microsoft.DotNet.Maestro.Client.Models.AssetData; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal abstract class PullRequestActorTests : SubscriptionOrPullRequestActorTests +{ + private const long InstallationId = 1174; + protected const string InProgressPrUrl = "https://github.com/owner/repo/pull/10"; + protected const string InProgressPrHeadBranch = "pr.head.branch"; + + private string _newBranch = null!; + + protected override void RegisterServices(IServiceCollection services) + { + base.RegisterServices(services); + + services.AddGitHubTokenProvider(); + services.AddSingleton(); + services.AddScoped(); + services.AddTransient(); + services.AddSingleton(MergePolicyEvaluator.Object); + services.AddSingleton(UpdateResolver.Object); + } + + protected override Task BeforeExecute(IServiceProvider context) + { + var dbContext = context.GetRequiredService(); + dbContext.Repositories.Add( + new Repository + { + RepositoryName = TargetRepo, + InstallationId = InstallationId + }); + return base.BeforeExecute(context); + } + + protected void ThenGetRequiredUpdatesShouldHaveBeenCalled(Build withBuild, bool prExists) + { + var assets = new List>(); + var dependencies = new List>(); + + UpdateResolver + .Verify(r => r.GetRequiredNonCoherencyUpdates(SourceRepo, NewCommit, Capture.In(assets), Capture.In(dependencies))); + + DarcRemotes[TargetRepo] + .Verify(r => r.GetDependenciesAsync(TargetRepo, prExists ? InProgressPrHeadBranch : TargetBranch, null)); + + UpdateResolver + .Verify(r => r.GetRequiredCoherencyUpdatesAsync(Capture.In(dependencies), RemoteFactory.Object)); + + assets.Should() + .BeEquivalentTo( + new List> + { + withBuild.Assets + .Select(a => new AssetData(false) + { + Name = a.Name, + Version = a.Version + }) + .ToList() + }); + } + + protected void AndCreateNewBranchShouldHaveBeenCalled() + { + var captureNewBranch = new CaptureMatch(newBranch => _newBranch = newBranch); + DarcRemotes[TargetRepo] + .Verify(r => r.CreateNewBranchAsync(TargetRepo, TargetBranch, Capture.With(captureNewBranch))); + } + + protected void AndCommitUpdatesShouldHaveBeenCalled(Build withUpdatesFromBuild) + { + var updatedDependencies = new List>(); + DarcRemotes[TargetRepo] + .Verify( + r => r.CommitUpdatesAsync( + TargetRepo, + _newBranch ?? InProgressPrHeadBranch, + RemoteFactory.Object, + It.IsAny(), + Capture.In(updatedDependencies), + It.IsAny())); + + updatedDependencies.Should() + .BeEquivalentTo( + new List> + { + withUpdatesFromBuild.Assets + .Select(a => new DependencyDetail + { + Name = a.Name, + Version = a.Version, + RepoUri = withUpdatesFromBuild.GitHubRepository, + Commit = "sha3" + }) + .ToList() + }); + } + + protected void AndCreatePullRequestShouldHaveBeenCalled() + { + var pullRequests = new List(); + DarcRemotes[TargetRepo] + .Verify(r => r.CreatePullRequestAsync(TargetRepo, Capture.In(pullRequests))); + + pullRequests.Should() + .BeEquivalentTo( + new List + { + new() + { + BaseBranch = TargetBranch, + HeadBranch = _newBranch + } + }, + options => options.Excluding(pr => pr.Title).Excluding(pr => pr.Description)); + + ValidatePRDescriptionContainsLinks(pullRequests[0]); + } + + protected void AndCodeFlowPullRequestShouldHaveBeenCreated() + { + var pullRequests = new List(); + DarcRemotes[TargetRepo] + .Verify(r => r.CreatePullRequestAsync(TargetRepo, Capture.In(pullRequests))); + + pullRequests.Should() + .BeEquivalentTo( + new List + { + new() + { + BaseBranch = TargetBranch, + HeadBranch = InProgressPrHeadBranch, + } + }, + options => options + .Excluding(pr => pr.Title) + .Excluding(pr => pr.Description)); + } + + protected void ThenPcsShouldNotHaveBeenCalled(Build build, string? prUrl = null) + { + CodeFlowWorkItemsProduced + .Should() + .NotContain(request => request.BuildId == build.Id && (prUrl == null || request.PrUrl == prUrl)); + } + + protected void ThenPcsShouldHaveBeenCalled(Build build, string? prUrl, out string prBranch) + => AndPcsShouldHaveBeenCalled(build, prUrl, out prBranch); + + protected void AndPcsShouldHaveBeenCalled(Build build, string? prUrl, out string prBranch) + { + var workItem = CodeFlowWorkItemsProduced + .FirstOrDefault(request => request.SubscriptionId == Subscription.Id && request.BuildId == build.Id && (prUrl == null || request.PrUrl == prUrl)); + + workItem.Should().NotBeNull(); + prBranch = workItem!.PrBranch; + } + + protected static void ValidatePRDescriptionContainsLinks(PullRequest pr) + { + pr.Description.Should().Contain("][1]"); + pr.Description.Should().Contain("[1]:"); + } + + protected void CreatePullRequestShouldReturnAValidValue() + { + DarcRemotes[TargetRepo] + .Setup(s => s.CreatePullRequestAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(InProgressPrUrl); + } + + protected void WithExistingPrBranch() + { + DarcRemotes[TargetRepo] + .Setup(s => s.BranchExistsAsync(TargetRepo, It.IsAny())) + .ReturnsAsync(true); + } + + protected void WithoutExistingPrBranch() + { + DarcRemotes[TargetRepo] + .Setup(s => s.BranchExistsAsync(TargetRepo, It.IsAny())) + .ReturnsAsync(false); + } + + protected void AndUpdatePullRequestShouldHaveBeenCalled() + { + var pullRequests = new List(); + DarcRemotes[TargetRepo] + .Verify(r => r.UpdatePullRequestAsync(InProgressPrUrl, Capture.In(pullRequests))); + pullRequests.Should() + .BeEquivalentTo( + new List + { + new() + { + BaseBranch = TargetBranch, + HeadBranch = _newBranch ?? InProgressPrHeadBranch + } + }, + options => options.Excluding(pr => pr.Title).Excluding(pr => pr.Description)); + } + + protected void AndSubscriptionShouldBeUpdatedForMergedPullRequest(Build withBuild) + { + Subscription.LastAppliedBuildId.Should().Be(withBuild.Id); + } + + protected void WithRequireNonCoherencyUpdates() + { + UpdateResolver + .Setup(r => r.GetRequiredNonCoherencyUpdates( + SourceRepo, + NewCommit, + It.IsAny>(), + It.IsAny>())) + .Returns( + (string sourceRepo, string sourceSha, IEnumerable assets, IEnumerable dependencies) => + // Just make from->to identical. + assets + .Select(d => new DependencyUpdate + { + From = new DependencyDetail + { + Name = d.Name, + Version = d.Version, + RepoUri = sourceRepo, + Commit = sourceSha + }, + To = new DependencyDetail + { + Name = d.Name, + Version = d.Version, + RepoUri = sourceRepo, + Commit = "sha3" + }, + }) + .ToList()); + } + + protected void WithNoRequiredCoherencyUpdates() + { + UpdateResolver + .Setup(r => r.GetRequiredCoherencyUpdatesAsync( + It.IsAny>(), + It.IsAny())) + .ReturnsAsync([]); + } + + protected void WithFailsStrictCheckForCoherencyUpdates() + { + UpdateResolver + .Setup(r => r.GetRequiredCoherencyUpdatesAsync( + It.IsAny>(), + It.IsAny())) + .ReturnsAsync( + (IEnumerable dependencies, IRemoteFactory factory) => + { + var fakeCoherencyError = new CoherencyError() + { + Dependency = new DependencyDetail() { Name = "fakeDependency" }, + Error = "Repo @ commit does not contain dependency fakeDependency", + PotentialSolutions = new List() + }; + throw new DarcCoherencyException(fakeCoherencyError); + }); + } + + protected void WithExistingPullRequest(Build forBuild, bool canUpdate) + { + AfterDbUpdateActions.Add(() => + { + var pr = CreatePullRequestCheckReminder(forBuild); + SetState(Subscription, pr); + SetExpectedState(Subscription, pr); + }); + + var remote = DarcRemotes.GetOrAddValue(TargetRepo, () => CreateMock()); + DarcRemotes[TargetRepo] + .Setup(x => x.GetPullRequestStatusAsync(InProgressPrUrl)) + .ReturnsAsync(PrStatus.Open); + + DarcRemotes[TargetRepo] + .Setup(r => r.GetPullRequestAsync(InProgressPrUrl)) + .ReturnsAsync( + new PullRequest + { + HeadBranch = InProgressPrHeadBranch, + BaseBranch = TargetBranch + }); + + var results = canUpdate + ? new MergePolicyEvaluationResults([]) + : new MergePolicyEvaluationResults( + [ + new MergePolicyEvaluationResult( + MergePolicyEvaluationStatus.Pending, + "Check", + "Fake one", + Mock.Of(x => x.Name == "Policy" && x.DisplayName == "Some policy")) + ]); + + MergePolicyEvaluator + .Setup(x => x.EvaluateAsync( + It.Is(pr => pr.Url == InProgressPrUrl), + It.IsAny(), + It.IsAny>())) + .ReturnsAsync(results); + } + + protected void WithExistingCodeFlowPullRequest(Build forBuild, bool canUpdate) + { + AfterDbUpdateActions.Add(() => + { + var pr = CreatePullRequestCheckReminder(forBuild); + SetState(Subscription, pr); + SetExpectedState(Subscription, pr); + }); + + DarcRemotes[TargetRepo] + .Setup(x => x.GetPullRequestStatusAsync(InProgressPrUrl)) + .ReturnsAsync(PrStatus.Open); + + var results = canUpdate + ? new MergePolicyEvaluationResults([]) + : new MergePolicyEvaluationResults( + [ + new MergePolicyEvaluationResult( + MergePolicyEvaluationStatus.Pending, + "Check", + "Fake one", + Mock.Of(x => x.Name == "Policy" && x.DisplayName == "Some policy")) + ]); + + MergePolicyEvaluator + .Setup(x => x.EvaluateAsync( + It.Is(pr => pr.Url == InProgressPrUrl), + It.IsAny(), + It.IsAny>())) + .ReturnsAsync(results); + } + + protected void WithExistingCodeFlowStatus(Build build) + { + AfterDbUpdateActions.Add(() => + { + SetState(Subscription, new CodeFlowStatus + { + PrBranch = InProgressPrHeadBranch, + SourceSha = build.Commit, + }); + }); + } + + protected void AndShouldHavePullRequestCheckReminder(Build forBuild, InProgressPullRequest? expectedState = null) + { + SetExpectedReminder(Subscription, expectedState ?? CreatePullRequestCheckReminder(forBuild)); + } + + protected void AndShouldHaveInProgressPullRequestState( + Build forBuild, + bool? coherencyCheckSuccessful = true, + List? coherencyErrors = null, + InProgressPullRequest? expectedState = null) + { + SetExpectedState(Subscription, expectedState ?? CreatePullRequestCheckReminder(forBuild, coherencyCheckSuccessful, coherencyErrors)); + } + + protected void ThenShouldHaveInProgressPullRequestState(Build forBuild, InProgressPullRequest? expectedState = null) + => AndShouldHaveInProgressPullRequestState(forBuild, expectedState: expectedState); + + protected void AndShouldHaveCodeFlowState(Build forBuild, string? prBranch = null) + { + SetExpectedState(Subscription, new CodeFlowStatus + { + SourceSha = forBuild.Commit, + PrBranch = prBranch, + }); + } + + protected void AndShouldHaveNoPendingUpdateState() + { + RemoveExpectedReminder(Subscription); + } + + protected virtual void ThenShouldHavePendingUpdateState(Build forBuild, bool isCodeFlow = false) + { + SetExpectedReminder(Subscription, CreateSubscriptionUpdate(forBuild, isCodeFlow)); + } + + protected void AndPendingUpdateIsRemoved() + { + RemoveExpectedReminder(Subscription); + } + + protected void ThenUpdateReminderIsRemoved() + { + RemoveExpectedReminder(Subscription); + } + + protected IPullRequestActor CreatePullRequestActor(IServiceProvider context) + { + var actorFactory = context.GetRequiredService(); + return actorFactory.CreatePullRequestActor(GetPullRequestActorId()); + } + + protected SubscriptionUpdateWorkItem CreateSubscriptionUpdate(Build forBuild, bool isCodeFlow = false) + => new() + { + ActorId = GetPullRequestActorId().ToString(), + SubscriptionId = Subscription.Id, + SubscriptionType = isCodeFlow ? SubscriptionType.DependenciesAndSources : SubscriptionType.Dependencies, + BuildId = forBuild.Id, + SourceSha = forBuild.Commit, + SourceRepo = forBuild.GitHubRepository ?? forBuild.AzureDevOpsRepository, + Assets = forBuild.Assets + .Select(a => new Asset + { + Name = a.Name, + Version = a.Version + }) + .ToList(), + IsCoherencyUpdate = false, + }; + + protected InProgressPullRequest CreatePullRequestCheckReminder( + Build forBuild, + bool? coherencyCheckSuccessful = true, + List? coherencyErrors = null) + => new() + { + ActorId = GetPullRequestActorId().ToString(), + ContainedSubscriptions = + [ + new SubscriptionPullRequestUpdate + { + BuildId = forBuild.Id, + SubscriptionId = Subscription.Id + } + ], + RequiredUpdates = forBuild.Assets + .Select(d => new DependencyUpdateSummary + { + DependencyName = d.Name, + FromVersion = d.Version, + ToVersion = d.Version + }) + .ToList(), + CoherencyCheckSuccessful = coherencyCheckSuccessful, + CoherencyErrors = coherencyErrors, + Url = InProgressPrUrl, + }; +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs index 2564a3ee29..dc3c75818c 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs @@ -110,8 +110,7 @@ public async Task ShouldReturnCorrectPRDescriptionForCodeEnabledSubscription() Build build = GivenANewBuildId(101, "abc1"); build.GitHubBranch = "main"; build.AzureDevOpsBuildNumber = "20230205.2"; - SubscriptionUpdateWorkItem update = GivenSubscriptionUpdate(false, build.Id, guid: "11111111-1111-1111-1111-111111111111"); - update.SubscriptionType = SubscriptionType.DependenciesAndSources; + SubscriptionUpdateWorkItem update = GivenSubscriptionUpdate(false, build.Id, guid: "11111111-1111-1111-1111-111111111111", SubscriptionType.DependenciesAndSources); string? description = null; await Execute( @@ -239,12 +238,18 @@ private List GivenDependencyUpdates(char version, int buildId) return dependencies; } - private static SubscriptionUpdateWorkItem GivenSubscriptionUpdate(bool isCoherencyUpdate, int buildId, string guid) => new() + private static SubscriptionUpdateWorkItem GivenSubscriptionUpdate( + bool isCoherencyUpdate, + int buildId, + string guid, + SubscriptionType type = SubscriptionType.Dependencies) => new() { + ActorId = guid, IsCoherencyUpdate = isCoherencyUpdate, SourceRepo = "The best repo", SubscriptionId = new Guid(guid), BuildId = buildId, + SubscriptionType = type, }; private static string BuildCorrectPRDescriptionWhenCoherencyUpdate(List deps) diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs index ba70c8ce79..38cd93d29c 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs @@ -198,6 +198,7 @@ private InProgressPullRequest GetInProgressPullRequest(string url, int contained // the "Url" and ContainedSubscriptions fields in InProgressPullRequestObjects return new InProgressPullRequest() { + ActorId = new BatchedPullRequestActorId(FakeRepoName, "main").Id, Url = url, ContainedSubscriptions = containedSubscriptions, SourceRepoNotified = false @@ -217,6 +218,7 @@ private InProgressPullRequest GetInProgressPullRequestWithoutTags(string url) return new InProgressPullRequest() { + ActorId = new BatchedPullRequestActorId(FakeRepoName, "main").Id, Url = url, ContainedSubscriptions = containedSubscriptions, SourceRepoNotified = false diff --git a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs index fd593dc18e..be83d944ee 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs @@ -122,22 +122,23 @@ internal Build GivenANewBuild(bool addToChannel, (string name, string version, b AzureDevOpsRepository = SourceRepo, Commit = NewCommit, DateProduced = DateTimeOffset.UtcNow, - Assets = new List( - assets.Select( - a => new Asset - { - Name = a.name, - Version = a.version, - NonShipping = a.nonShipping, - Locations = - [ - new AssetLocation - { - Location = AssetFeedUrl, - Type = LocationType.NugetFeed - } - ] - })) + Assets = + [ + ..assets.Select(a => new Asset + { + Name = a.name, + Version = a.version, + NonShipping = a.nonShipping, + Locations = + [ + new AssetLocation + { + Location = AssetFeedUrl, + Type = LocationType.NugetFeed + } + ] + }) + ] }; ContextUpdates.Add( context => diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs new file mode 100644 index 0000000000..da3a17227f --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +/// +/// Tests the code flow PR update logic. +/// The tests are writter in the order in which the different phases of the PR are written. +/// Each test should have the inner state that is left behind by the previous state. +/// +[TestFixture, NonParallelizable] +internal class UpdateAssetsForCodeFlowTests : UpdateAssetsPullRequestActorTests +{ + [Test] + public async Task UpdateWithNoExistingStateOrPrBranch() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + Build build = GivenANewBuild(true); + + await WhenUpdateAssetsAsyncIsCalled(build); + + ThenShouldHavePendingUpdateState(build); + AndPcsShouldHaveBeenCalled(build, prUrl: null, out var requestedBranch); + AndShouldHaveCodeFlowState(build, requestedBranch); + } + + [Test] + public async Task WaitForPrBranch() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + Build build = GivenANewBuild(true); + + GivenPendingUpdates(build); + WithExistingCodeFlowStatus(build); + WithoutExistingPrBranch(); + + await WhenUpdateAssetsAsyncIsCalled(build); + + ThenShouldHavePendingUpdateState(build); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + } + + [Test] + public async Task UpdateWithPrBranchReady() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + Build build = GivenANewBuild(true); + + GivenPendingUpdates(build); + WithExistingCodeFlowStatus(build); + WithExistingPrBranch(); + CreatePullRequestShouldReturnAValidValue(); + + await WhenUpdateAssetsAsyncIsCalled(build); + + // TODO (https://github.com/dotnet/arcade-services/issues/3866): We need to populate InProgressPullRequest fully + // with assets and other info just like we do in UpdatePullRequestAsync. + // Right now, we are not flowing packages in codeflow subscriptions yet, so this functionality is no there + // For now, we manually update the info the unit tests expect. + var expectedState = new WorkItems.InProgressPullRequest() + { + ActorId = GetPullRequestActorId(Subscription).Id, + Url = InProgressPrUrl, + ContainedSubscriptions = + [ + new() + { + SubscriptionId = Subscription.Id, + BuildId = build.Id, + } + ] + }; + + ThenUpdateReminderIsRemoved(); + ThenPcsShouldNotHaveBeenCalled(build); + AndCodeFlowPullRequestShouldHaveBeenCreated(); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + AndShouldHavePullRequestCheckReminder(build, expectedState); + AndShouldHaveInProgressPullRequestState(build, coherencyCheckSuccessful: null, expectedState: expectedState); + AndPendingUpdateIsRemoved(); + } + + [Test] + public async Task UpdateWithPrNotUpdatable() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + Build build = GivenANewBuild(true); + + WithExistingCodeFlowStatus(build); + WithExistingPrBranch(); + WithExistingPullRequest(build, canUpdate: false); + + await WhenUpdateAssetsAsyncIsCalled(build); + + ThenShouldHavePendingUpdateState(build); + ThenPcsShouldNotHaveBeenCalled(build, InProgressPrUrl); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + AndShouldHaveInProgressPullRequestState(build, coherencyCheckSuccessful: true); + AndShouldHavePullRequestCheckReminder(build); + } + + [Test] + public async Task UpdateWithPrUpdatableButNoUpdates() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + Build build = GivenANewBuild(true); + + WithExistingCodeFlowStatus(build); + WithExistingPrBranch(); + WithExistingPullRequest(build, canUpdate: true); + + await WhenUpdateAssetsAsyncIsCalled(build); + + ThenPcsShouldNotHaveBeenCalled(build, InProgressPrUrl); + AndShouldHaveCodeFlowState(build, InProgressPrHeadBranch); + AndShouldHavePullRequestCheckReminder(build); + AndShouldHaveInProgressPullRequestState(build); + } + + [Test] + public async Task UpdateCodeFlowPrWithNewBuild() + { + GivenATestChannel(); + GivenACodeFlowSubscription( + new SubscriptionPolicy + { + Batchable = false, + UpdateFrequency = UpdateFrequency.EveryBuild, + }); + + Build oldBuild = GivenANewBuild(true); + Build newBuild = GivenANewBuild(true); + newBuild.Commit = "sha456"; + + WithExistingCodeFlowStatus(oldBuild); + WithExistingPrBranch(); + WithExistingPullRequest(oldBuild, canUpdate: true); + + await WhenUpdateAssetsAsyncIsCalled(newBuild); + + ThenPcsShouldHaveBeenCalled(newBuild, InProgressPrUrl, out _); + AndShouldHaveCodeFlowState(newBuild, InProgressPrHeadBranch); + AndShouldHavePullRequestCheckReminder(newBuild); + AndShouldHaveInProgressPullRequestState(newBuild); + } + + protected override void ThenShouldHavePendingUpdateState(Build forBuild, bool _ = false) + { + base.ThenShouldHavePendingUpdateState(forBuild, true); + } + + protected void GivenPendingUpdates(Build forBuild) + { + AfterDbUpdateActions.Add( + () => + { + var update = CreateSubscriptionUpdate(forBuild, isCodeFlow: false); + SetExpectedReminder(Subscription, update); + }); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs new file mode 100644 index 0000000000..75a0c0a4bc --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Data.Models; +using Asset = Maestro.Contracts.Asset; + +namespace ProductConstructionService.DependencyFlow.Tests; + +internal abstract class UpdateAssetsPullRequestActorTests : PullRequestActorTests +{ + protected async Task WhenUpdateAssetsAsyncIsCalled(Build forBuild) + { + await Execute( + async context => + { + IPullRequestActor actor = CreatePullRequestActor(context); + await actor.UpdateAssetsAsync( + Subscription.Id, + Subscription.SourceEnabled ? SubscriptionType.DependenciesAndSources : SubscriptionType.Dependencies, + forBuild.Id, + SourceRepo, + forBuild.Commit, + forBuild.Assets + .Select(a => new Asset + { + Name = a.Name, + Version = a.Version + }) + .ToList()); + }); + } +} diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs new file mode 100644 index 0000000000..0fd3e07857 --- /dev/null +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Contracts; +using Maestro.Data.Models; +using NUnit.Framework; + +namespace ProductConstructionService.DependencyFlow.Tests; + +[TestFixture, NonParallelizable] +internal class UpdateAssetsTests : UpdateAssetsPullRequestActorTests +{ + [TestCase(false)] + [TestCase(true)] + public async Task UpdateWithAssetsNoExistingPR(bool batchable) + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = batchable, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + WithRequireNonCoherencyUpdates(); + WithNoRequiredCoherencyUpdates(); + + CreatePullRequestShouldReturnAValidValue(); + + await WhenUpdateAssetsAsyncIsCalled(b); + + ThenGetRequiredUpdatesShouldHaveBeenCalled(b, false); + AndCreateNewBranchShouldHaveBeenCalled(); + AndCommitUpdatesShouldHaveBeenCalled(b); + AndCreatePullRequestShouldHaveBeenCalled(); + AndShouldHavePullRequestCheckReminder(b); + AndShouldHaveInProgressPullRequestState(b); + } + + [TestCase(false)] + [TestCase(true)] + public async Task UpdateWithAssetsExistingPR(bool batchable) + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = batchable, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + WithRequireNonCoherencyUpdates(); + WithNoRequiredCoherencyUpdates(); + WithExistingPullRequest(b, canUpdate: true); + + await WhenUpdateAssetsAsyncIsCalled(b); + + ThenGetRequiredUpdatesShouldHaveBeenCalled(b, true); + AndCommitUpdatesShouldHaveBeenCalled(b); + AndUpdatePullRequestShouldHaveBeenCalled(); + AndShouldHavePullRequestCheckReminder(b); + AndShouldHaveInProgressPullRequestState(b); + } + + [TestCase(false)] + [TestCase(true)] + public async Task UpdateWithAssetsExistingPRNotUpdatable(bool batchable) + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = batchable, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + WithRequireNonCoherencyUpdates(); + WithNoRequiredCoherencyUpdates(); + WithExistingPullRequest(b, canUpdate: false); + + await WhenUpdateAssetsAsyncIsCalled(b); + + ThenShouldHavePendingUpdateState(b); + AndShouldHaveInProgressPullRequestState(b); + AndShouldHavePullRequestCheckReminder(b); + } + + [TestCase(false)] + [TestCase(true)] + public async Task UpdateWithNoAssets(bool batchable) + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = batchable, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true, Array.Empty<(string, string, bool)>()); + + WithRequireNonCoherencyUpdates(); + WithNoRequiredCoherencyUpdates(); + + await WhenUpdateAssetsAsyncIsCalled(b); + + ThenGetRequiredUpdatesShouldHaveBeenCalled(b, false); + AndSubscriptionShouldBeUpdatedForMergedPullRequest(b); + } + + [TestCase(false)] + [TestCase(true)] + public async Task UpdateWithAssetsWhenStrictAlgorithmFails(bool batchable) + { + GivenATestChannel(); + GivenASubscription( + new SubscriptionPolicy + { + Batchable = batchable, + UpdateFrequency = UpdateFrequency.EveryBuild + }); + Build b = GivenANewBuild(true); + + WithRequireNonCoherencyUpdates(); + WithFailsStrictCheckForCoherencyUpdates(); + + CreatePullRequestShouldReturnAValidValue(); + + await WhenUpdateAssetsAsyncIsCalled(b); + + ThenGetRequiredUpdatesShouldHaveBeenCalled(b, false); + AndCreateNewBranchShouldHaveBeenCalled(); + AndCommitUpdatesShouldHaveBeenCalled(b); + AndCreatePullRequestShouldHaveBeenCalled(); + AndShouldHavePullRequestCheckReminder(b, CreatePullRequestCheckReminder(b, + coherencyCheckSuccessful: false, + coherencyErrors: [ + new CoherencyErrorDetails() + { + Error = "Repo @ commit does not contain dependency fakeDependency", + PotentialSolutions = new List() + } + ])); + AndShouldHaveInProgressPullRequestState(b, + coherencyCheckSuccessful: false, + coherencyErrors: [ + new CoherencyErrorDetails() + { + Error = "Repo @ commit does not contain dependency fakeDependency", + PotentialSolutions = new List() + } + ]); + } +} diff --git a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs index 61c2fc0b27..c0478180eb 100644 --- a/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs +++ b/test/ProductConstructionService.SubscriptionTriggerer.Tests/SubscriptionTriggererTests.cs @@ -21,21 +21,21 @@ public class SubscriptionTriggererTests private BuildAssetRegistryContext? _context; private ServiceProvider? _provider; private IServiceScope _scope = new Mock().Object; - private List _updateSubscriptionWorkItems = []; + private List _updateSubscriptionWorkItems = []; [SetUp] public void Setup() { var services = new ServiceCollection(); - _updateSubscriptionWorkItems = new(); + _updateSubscriptionWorkItems = []; Mock workItemProducerFactoryMock = new(); - Mock> workItemProducerMock = new(); + Mock> workItemProducerMock = new(); - workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) + workItemProducerMock.Setup(w => w.ProduceWorkItemAsync(It.IsAny(), TimeSpan.Zero)) .ReturnsAsync(QueuesModelFactory.SendReceipt("message", DateTimeOffset.Now, DateTimeOffset.Now, "popReceipt", DateTimeOffset.Now)) - .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); - workItemProducerFactoryMock.Setup(w => w.CreateProducer()) + .Callback((item, _) => _updateSubscriptionWorkItems.Add(item)); + workItemProducerFactoryMock.Setup(w => w.CreateProducer()) .Returns(workItemProducerMock.Object); services.AddLogging(); From 2b7f6272b17b1ec962103e4c901f6ba5c54a5774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 28 Aug 2024 16:13:31 +0200 Subject: [PATCH 10/33] Cover PR actors with DI tests (#3888) --- .../Properties/AssemblyInfo.cs | 1 + .../DependencyRegistrationTests.cs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs index 3eed7b6adb..61d5e2902c 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/Properties/AssemblyInfo.cs @@ -3,5 +3,6 @@ using System.Runtime.CompilerServices; +[assembly: InternalsVisibleTo("ProductConstructionService.Api.Tests")] [assembly: InternalsVisibleTo("ProductConstructionService.DependencyFlow.Tests")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] diff --git a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs index 8e149392d1..99331f9f22 100644 --- a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs @@ -5,6 +5,9 @@ using Microsoft.DotNet.Internal.DependencyInjection.Testing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using ProductConstructionService.DependencyFlow; +using ProductConstructionService.DependencyFlow.WorkItemProcessors; +using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -19,6 +22,11 @@ await builder.ConfigurePcs( authRedis: false, addSwagger: true); + builder.Services.AddTransient(); + builder.Services.AddSingleton(new NonBatchedPullRequestActorId(Guid.NewGuid())); + builder.Services.AddTransient(); + builder.Services.AddSingleton(new BatchedPullRequestActorId("repo", "branch")); + DependencyInjectionValidation.IsDependencyResolutionCoherent( s => { From 170cf53ad620422f82aaaa7df08e997b2258e0bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 28 Aug 2024 17:16:19 +0200 Subject: [PATCH 11/33] Rename actors to updaters (#3889) --- .../ActorFactory.cs | 35 ------------------ ...tActor.cs => BatchedPullRequestUpdater.cs} | 14 ++++---- .../DependencyFlowConfiguration.cs | 2 +- ...RequestActor.cs => IPullRequestUpdater.cs} | 2 +- ...tionActor.cs => ISubscriptionTriggerer.cs} | 3 +- ...tor.cs => NonBatchedPullRequestUpdater.cs} | 14 ++++---- ...lRequestActor.cs => PullRequestUpdater.cs} | 32 ++++++++--------- .../PullRequestUpdaterFactory.cs | 35 ++++++++++++++++++ ...uestActorId.cs => PullRequestUpdaterId.cs} | 30 ++++++++-------- ...ptionActor.cs => SubscriptionTriggerer.cs} | 28 +++++++-------- .../PullRequestCheckProcessor.cs | 10 +++--- .../SubscriptionTriggerProcessor.cs | 10 +++--- .../SubscriptionUpdateProcessor.cs | 10 +++--- .../DependencyRegistrationTests.cs | 10 +++--- .../PendingCodeFlowUpdatesTests.cs | 2 +- ...> PendingUpdatePullRequestUpdaterTests.cs} | 6 ++-- .../PendingUpdatesTests.cs | 2 +- .../PullRequestBuilderTests.cs | 2 +- .../PullRequestPolicyFailureNotifierTests.cs | 4 +-- ...torTests.cs => PullRequestUpdaterTests.cs} | 12 +++---- ... SubscriptionOrPullRequestUpdaterTests.cs} | 8 ++--- ...orTests.cs => SubscriptionUpdaterTests.cs} | 36 +++++++++---------- .../UpdateAssetsForCodeFlowTests.cs | 4 +-- ...=> UpdateAssetsPullRequestUpdaterTests.cs} | 6 ++-- .../UpdateAssetsTests.cs | 2 +- .../{ActorTests.cs => UpdaterTests.cs} | 36 +++++++++---------- 26 files changed, 175 insertions(+), 180 deletions(-) delete mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{BatchedPullRequestActor.cs => BatchedPullRequestUpdater.cs} (85%) rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{IPullRequestActor.cs => IPullRequestUpdater.cs} (94%) rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{ISubscriptionActor.cs => ISubscriptionTriggerer.cs} (87%) rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{NonBatchedPullRequestActor.cs => NonBatchedPullRequestUpdater.cs} (91%) rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{PullRequestActor.cs => PullRequestUpdater.cs} (97%) create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterFactory.cs rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{PullRequestActorId.cs => PullRequestUpdaterId.cs} (63%) rename src/ProductConstructionService/ProductConstructionService.DependencyFlow/{SubscriptionActor.cs => SubscriptionTriggerer.cs} (83%) rename test/ProductConstructionService.DependencyFlow.Tests/{PendingUpdatePullRequestActorTests.cs => PendingUpdatePullRequestUpdaterTests.cs} (81%) rename test/ProductConstructionService.DependencyFlow.Tests/{PullRequestActorTests.cs => PullRequestUpdaterTests.cs} (97%) rename test/ProductConstructionService.DependencyFlow.Tests/{SubscriptionOrPullRequestActorTests.cs => SubscriptionOrPullRequestUpdaterTests.cs} (94%) rename test/ProductConstructionService.DependencyFlow.Tests/{SubscriptionActorTests.cs => SubscriptionUpdaterTests.cs} (65%) rename test/ProductConstructionService.DependencyFlow.Tests/{UpdateAssetsPullRequestActorTests.cs => UpdateAssetsPullRequestUpdaterTests.cs} (81%) rename test/ProductConstructionService.DependencyFlow.Tests/{ActorTests.cs => UpdaterTests.cs} (80%) diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs deleted file mode 100644 index 5bb9581d9e..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ActorFactory.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Extensions.DependencyInjection; - -namespace ProductConstructionService.DependencyFlow; - -public interface IActorFactory -{ - IPullRequestActor CreatePullRequestActor(PullRequestActorId actorId); - - ISubscriptionActor CreateSubscriptionActor(Guid subscriptionId); -} - -internal class ActorFactory : IActorFactory -{ - private readonly IServiceProvider _serviceProvider; - - public ActorFactory(IServiceProvider serviceProvider) - { - _serviceProvider = serviceProvider; - } - - public IPullRequestActor CreatePullRequestActor(PullRequestActorId actorId) => actorId switch - { - BatchedPullRequestActorId batched => ActivatorUtilities.CreateInstance(_serviceProvider, batched), - NonBatchedPullRequestActorId nonBatched => ActivatorUtilities.CreateInstance(_serviceProvider, nonBatched), - _ => throw new NotImplementedException() - }; - - public ISubscriptionActor CreateSubscriptionActor(Guid subscriptionId) - { - return ActivatorUtilities.CreateInstance(_serviceProvider, subscriptionId); - } -} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestUpdater.cs similarity index 85% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestUpdater.cs index 26f5a042d7..68133be9d8 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/BatchedPullRequestUpdater.cs @@ -14,28 +14,28 @@ namespace ProductConstructionService.DependencyFlow; /// A for batched subscriptions that reads its Target and Merge Policies /// from the configuration for a repository /// -internal class BatchedPullRequestActor : PullRequestActor +internal class BatchedPullRequestUpdater : PullRequestUpdater { - private readonly BatchedPullRequestActorId _id; + private readonly BatchedPullRequestUpdaterId _id; private readonly BuildAssetRegistryContext _context; - public BatchedPullRequestActor( - BatchedPullRequestActorId id, + public BatchedPullRequestUpdater( + BatchedPullRequestUpdaterId id, IMergePolicyEvaluator mergePolicyEvaluator, BuildAssetRegistryContext context, IRemoteFactory remoteFactory, - IActorFactory actorFactory, + IPullRequestUpdaterFactory updaterFactory, ICoherencyUpdateResolver coherencyUpdateResolver, IPullRequestBuilder pullRequestBuilder, IRedisCacheFactory cacheFactory, IReminderManagerFactory reminderManagerFactory, IWorkItemProducerFactory workItemProducerFactory, - ILogger logger) + ILogger logger) : base( id, mergePolicyEvaluator, remoteFactory, - actorFactory, + updaterFactory, coherencyUpdateResolver, pullRequestBuilder, cacheFactory, diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs index d885e19c0c..5ddd591910 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/DependencyFlowConfiguration.cs @@ -19,7 +19,7 @@ public static void AddDependencyFlowProcessors(this IHostApplicationBuilder buil public static void AddDependencyFlowProcessors(this IServiceCollection services) { - services.TryAddTransient(); + services.TryAddTransient(); services.TryAddSingleton(); services.TryAddTransient(); services.TryAddTransient(); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestUpdater.cs similarity index 94% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestUpdater.cs index a00ba43d62..e40f1790c8 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/IPullRequestUpdater.cs @@ -5,7 +5,7 @@ namespace ProductConstructionService.DependencyFlow; -public interface IPullRequestActor +public interface IPullRequestUpdater { Task SynchronizeInProgressPullRequestAsync( InProgressPullRequest pullRequestCheck); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionTriggerer.cs similarity index 87% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionTriggerer.cs index 8969b35ab6..fb51339bfa 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/ISubscriptionTriggerer.cs @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using Maestro.Contracts; -using Microsoft.ServiceFabric.Actors; namespace ProductConstructionService.DependencyFlow; -public interface ISubscriptionActor : IActor +public interface ISubscriptionTriggerer { Task UpdateSubscriptionAsync(int buildId); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs similarity index 91% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs index 46c2876b7c..25d0feea2e 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/NonBatchedPullRequestUpdater.cs @@ -15,31 +15,31 @@ namespace ProductConstructionService.DependencyFlow; /// A that reads its Merge Policies and Target information from a /// non-batched subscription object /// -internal class NonBatchedPullRequestActor : PullRequestActor +internal class NonBatchedPullRequestUpdater : PullRequestUpdater { private readonly Lazy> _lazySubscription; - private readonly NonBatchedPullRequestActorId _id; + private readonly NonBatchedPullRequestUpdaterId _id; private readonly BuildAssetRegistryContext _context; private readonly IPullRequestPolicyFailureNotifier _pullRequestPolicyFailureNotifier; - public NonBatchedPullRequestActor( - NonBatchedPullRequestActorId id, + public NonBatchedPullRequestUpdater( + NonBatchedPullRequestUpdaterId id, IMergePolicyEvaluator mergePolicyEvaluator, BuildAssetRegistryContext context, IRemoteFactory remoteFactory, - IActorFactory actorFactory, + IPullRequestUpdaterFactory updaterFactory, ICoherencyUpdateResolver coherencyUpdateResolver, IPullRequestBuilder pullRequestBuilder, IPullRequestPolicyFailureNotifier pullRequestPolicyFailureNotifier, IRedisCacheFactory cacheFactory, IReminderManagerFactory reminderManagerFactory, IWorkItemProducerFactory workItemProducerFactory, - ILogger logger) + ILogger logger) : base( id, mergePolicyEvaluator, remoteFactory, - actorFactory, + updaterFactory, coherencyUpdateResolver, pullRequestBuilder, cacheFactory, diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs similarity index 97% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index fe3788f648..486bc4fbc8 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -3,7 +3,6 @@ using System.Net; using Maestro.Contracts; -using Maestro.Data; using Maestro.Data.Models; using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib; @@ -18,17 +17,16 @@ namespace ProductConstructionService.DependencyFlow; /// -/// A service fabric actor implementation that is responsible for creating and updating pull requests for dependency -/// updates. +/// A class responsible for creating and updating pull requests for dependency updates. /// -internal abstract class PullRequestActor : IPullRequestActor +internal abstract class PullRequestUpdater : IPullRequestUpdater { private static readonly TimeSpan DefaultReminderDuration = TimeSpan.FromMinutes(5); - private readonly PullRequestActorId _id; + private readonly PullRequestUpdaterId _id; private readonly IMergePolicyEvaluator _mergePolicyEvaluator; private readonly IRemoteFactory _remoteFactory; - private readonly IActorFactory _actorFactory; + private readonly IPullRequestUpdaterFactory _updaterFactory; private readonly ICoherencyUpdateResolver _coherencyUpdateResolver; private readonly IPullRequestBuilder _pullRequestBuilder; // TODO (https://github.com/dotnet/arcade-services/issues/3866): When removed, remove the mocks from tests @@ -43,11 +41,11 @@ internal abstract class PullRequestActor : IPullRequestActor /// /// Creates a new PullRequestActor /// - public PullRequestActor( - PullRequestActorId id, + public PullRequestUpdater( + PullRequestUpdaterId id, IMergePolicyEvaluator mergePolicyEvaluator, IRemoteFactory remoteFactory, - IActorFactory actorFactory, + IPullRequestUpdaterFactory updaterFactory, ICoherencyUpdateResolver coherencyUpdateResolver, IPullRequestBuilder pullRequestBuilder, IRedisCacheFactory cacheFactory, @@ -58,7 +56,7 @@ public PullRequestActor( _id = id; _mergePolicyEvaluator = mergePolicyEvaluator; _remoteFactory = remoteFactory; - _actorFactory = actorFactory; + _updaterFactory = updaterFactory; _coherencyUpdateResolver = coherencyUpdateResolver; _pullRequestBuilder = pullRequestBuilder; _workItemProducerFactory = workItemProducerFactory; @@ -84,7 +82,7 @@ public async Task ProcessPendingUpdatesAsync(SubscriptionUpdateWorkItem up { _logger.LogInformation("Processing pending updates for subscription {subscriptionId}", update.SubscriptionId); - // Check if we have an on-going PR for this actor + // Check if we track an on-going PR already InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); if (pr == null) @@ -163,7 +161,7 @@ public virtual async Task SynchronizeInProgressPullRequestAsync(InProgress switch (result) { - // If the PR was merged or closed, we are done with it and the actor doesn't + // If the PR was merged or closed, we are done with it and we don't // need to periodically run the synchronization any longer. case PullRequestStatus.Completed: case PullRequestStatus.UnknownPR: @@ -346,8 +344,8 @@ private async Task UpdateSubscriptionsForMergedPRAsync(IEnumerable UpdateAssetsAsync( string sourceSha, List assets) { - // Check if we have an on-going PR for this actor + // Check if we track an on-going PR already InProgressPullRequest? pr = await _pullRequestState.TryGetStateAsync(); bool canUpdate; if (pr == null) diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterFactory.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterFactory.cs new file mode 100644 index 0000000000..3938d3c8cf --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterFactory.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.DependencyInjection; + +namespace ProductConstructionService.DependencyFlow; + +public interface IPullRequestUpdaterFactory +{ + IPullRequestUpdater CreatePullRequestUpdater(PullRequestUpdaterId updaterId); + + ISubscriptionTriggerer CreateSubscriptionTrigerrer(Guid subscriptionId); +} + +internal class PullRequestUpdaterFactory : IPullRequestUpdaterFactory +{ + private readonly IServiceProvider _serviceProvider; + + public PullRequestUpdaterFactory(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + public IPullRequestUpdater CreatePullRequestUpdater(PullRequestUpdaterId updaterId) => updaterId switch + { + BatchedPullRequestUpdaterId batched => ActivatorUtilities.CreateInstance(_serviceProvider, batched), + NonBatchedPullRequestUpdaterId nonBatched => ActivatorUtilities.CreateInstance(_serviceProvider, nonBatched), + _ => throw new NotImplementedException() + }; + + public ISubscriptionTriggerer CreateSubscriptionTrigerrer(Guid subscriptionId) + { + return ActivatorUtilities.CreateInstance(_serviceProvider, subscriptionId); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterId.cs similarity index 63% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterId.cs index 1e28bd8de1..6cac0f2b61 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestActorId.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdaterId.cs @@ -5,44 +5,44 @@ namespace ProductConstructionService.DependencyFlow; -public class ActorId +public class UpdaterId { public string Id { get; } - public ActorId(string id) + public UpdaterId(string id) { Id = id; } public override string ToString() => Id.ToString(); - public override bool Equals(object? obj) => Id.Equals((obj as ActorId)?.Id); + public override bool Equals(object? obj) => Id.Equals((obj as UpdaterId)?.Id); public override int GetHashCode() => Id.GetHashCode(); } -public class NonBatchedPullRequestActorId : PullRequestActorId +public class NonBatchedPullRequestUpdaterId : PullRequestUpdaterId { public Guid SubscriptionId { get; } - public NonBatchedPullRequestActorId(Guid subscriptionId) + public NonBatchedPullRequestUpdaterId(Guid subscriptionId) : base(subscriptionId.ToString()) { SubscriptionId = subscriptionId; } } -public class BatchedPullRequestActorId : PullRequestActorId +public class BatchedPullRequestUpdaterId : PullRequestUpdaterId { public string Repository { get; } public string Branch { get; } /// - /// Creates an identifying the PullRequestActor responsible for pull requests for all batched + /// Creates an identifying the PullRequestActor responsible for pull requests for all batched /// subscriptions /// targeting the (, ) pair. /// - public BatchedPullRequestActorId(string repository, string branch) + public BatchedPullRequestUpdaterId(string repository, string branch) : base(Encode(repository) + ":" + Encode(branch)) { Repository = repository; @@ -50,33 +50,33 @@ public BatchedPullRequestActorId(string repository, string branch) } } -public abstract class PullRequestActorId : ActorId +public abstract class PullRequestUpdaterId : UpdaterId { - protected PullRequestActorId(string id) + protected PullRequestUpdaterId(string id) : base(id) { } /// - /// Parses an created by into the (repository, branch) + /// Parses an created by into the (repository, branch) /// pair that created it. /// - public static PullRequestActorId Parse(string id) + public static PullRequestUpdaterId Parse(string id) { if (Guid.TryParse(id, out var guid)) { - return new NonBatchedPullRequestActorId(guid); + return new NonBatchedPullRequestUpdaterId(guid); } var colonIndex = id.IndexOf(":", StringComparison.Ordinal); if (colonIndex == -1) { - throw new ArgumentException("Actor id not in correct format", nameof(id)); + throw new ArgumentException("Updater ID not in correct format", nameof(id)); } var repository = Decode(id.Substring(0, colonIndex)); var branch = Decode(id.Substring(colonIndex + 1)); - return new BatchedPullRequestActorId(repository, branch); + return new BatchedPullRequestUpdaterId(repository, branch); } protected static string Encode(string repository) diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionTriggerer.cs similarity index 83% rename from src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs rename to src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionTriggerer.cs index 342ffb0e8c..0e84e20ffd 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionActor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/SubscriptionTriggerer.cs @@ -10,21 +10,21 @@ namespace ProductConstructionService.DependencyFlow; -internal class SubscriptionActor : ISubscriptionActor +internal class SubscriptionTriggerer : ISubscriptionTriggerer { - private readonly IActorFactory _actorFactory; + private readonly IPullRequestUpdaterFactory _updaterFactory; private readonly BuildAssetRegistryContext _context; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly Guid _subscriptionId; - public SubscriptionActor( + public SubscriptionTriggerer( BuildAssetRegistryContext context, - IActorFactory actorFactory, - ILogger logger, + IPullRequestUpdaterFactory updaterFactory, + ILogger logger, Guid subscriptionId) { _context = context; - _actorFactory = actorFactory; + _updaterFactory = updaterFactory; _logger = logger; _subscriptionId = subscriptionId; } @@ -117,24 +117,24 @@ await AddDependencyFlowEventAsync( .ThenInclude(a => a.Locations) .FirstAsync(b => b.Id == buildId); - IPullRequestActor pullRequestActor; + IPullRequestUpdater pullRequestActor; if (subscription.PolicyObject.Batchable) { - _logger.LogInformation("Creating pull request actor for branch {branch} of {repository}", + _logger.LogInformation("Creating pull request updater for branch {branch} of {repository}", subscription.TargetBranch, subscription.TargetRepository); - pullRequestActor = _actorFactory.CreatePullRequestActor( - new BatchedPullRequestActorId(subscription.TargetRepository, subscription.TargetBranch)); + pullRequestActor = _updaterFactory.CreatePullRequestUpdater( + new BatchedPullRequestUpdaterId(subscription.TargetRepository, subscription.TargetBranch)); } else { - _logger.LogInformation("Creating pull request actor for subscription {subscriptionId}", + _logger.LogInformation("Creating pull request updater for subscription {subscriptionId}", _subscriptionId); - pullRequestActor = _actorFactory.CreatePullRequestActor( - new NonBatchedPullRequestActorId(_subscriptionId)); + pullRequestActor = _updaterFactory.CreatePullRequestUpdater( + new NonBatchedPullRequestUpdaterId(_subscriptionId)); } var assets = build.Assets diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs index 644523b268..51f4b36414 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/PullRequestCheckProcessor.cs @@ -8,19 +8,19 @@ namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; public class PullRequestCheckProcessor : WorkItemProcessor { - private readonly IActorFactory _actorFactory; + private readonly IPullRequestUpdaterFactory _updaterFactory; - public PullRequestCheckProcessor(IActorFactory actorFactory) + public PullRequestCheckProcessor(IPullRequestUpdaterFactory updaterFactory) { - _actorFactory = actorFactory; + _updaterFactory = updaterFactory; } public override async Task ProcessWorkItemAsync( InProgressPullRequest workItem, CancellationToken cancellationToken) { - var actor = _actorFactory.CreatePullRequestActor(PullRequestActorId.Parse(workItem.ActorId)); - await actor.SynchronizeInProgressPullRequestAsync(workItem); + var updater = _updaterFactory.CreatePullRequestUpdater(PullRequestUpdaterId.Parse(workItem.ActorId)); + await updater.SynchronizeInProgressPullRequestAsync(workItem); return true; } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs index a548fd86f9..b523f6a859 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionTriggerProcessor.cs @@ -13,18 +13,18 @@ public class SubscriptionTriggerProcessor : WorkItemProcessor _logger; public SubscriptionTriggerProcessor( BuildAssetRegistryContext context, OperationManager operationManager, - IActorFactory actorFactory, + IPullRequestUpdaterFactory updaterFactory, ILogger logger) { _context = context; _operations = operationManager; - _actorFactory = actorFactory; + _updaterFactory = updaterFactory; _logger = logger; } @@ -109,8 +109,8 @@ private async Task UpdateSubscriptionAsync(Guid subscriptionId, int buildI { try { - ISubscriptionActor actor = _actorFactory.CreateSubscriptionActor(subscriptionId); - await actor.UpdateSubscriptionAsync(buildId); + ISubscriptionTriggerer triggerer = _updaterFactory.CreateSubscriptionTrigerrer(subscriptionId); + await triggerer.UpdateSubscriptionAsync(buildId); return true; } catch (Exception e) diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs index ad2aaceb88..0741eb4078 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/WorkItemProcessors/SubscriptionUpdateProcessor.cs @@ -8,19 +8,19 @@ namespace ProductConstructionService.DependencyFlow.WorkItemProcessors; public class SubscriptionUpdateProcessor : WorkItemProcessor { - private readonly IActorFactory _actorFactory; + private readonly IPullRequestUpdaterFactory _updaterFactory; - public SubscriptionUpdateProcessor(IActorFactory actorFactory) + public SubscriptionUpdateProcessor(IPullRequestUpdaterFactory updaterFactory) { - _actorFactory = actorFactory; + _updaterFactory = updaterFactory; } public override async Task ProcessWorkItemAsync( SubscriptionUpdateWorkItem workItem, CancellationToken cancellationToken) { - var actor = _actorFactory.CreatePullRequestActor(PullRequestActorId.Parse(workItem.ActorId)); - await actor.ProcessPendingUpdatesAsync(workItem); + var updater = _updaterFactory.CreatePullRequestUpdater(PullRequestUpdaterId.Parse(workItem.ActorId)); + await updater.ProcessPendingUpdatesAsync(workItem); return true; } } diff --git a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs index 99331f9f22..7da74cf7a3 100644 --- a/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs +++ b/test/ProductConstructionService.Api.Tests/DependencyRegistrationTests.cs @@ -6,8 +6,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using ProductConstructionService.DependencyFlow; -using ProductConstructionService.DependencyFlow.WorkItemProcessors; -using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Tests; @@ -22,10 +20,10 @@ await builder.ConfigurePcs( authRedis: false, addSwagger: true); - builder.Services.AddTransient(); - builder.Services.AddSingleton(new NonBatchedPullRequestActorId(Guid.NewGuid())); - builder.Services.AddTransient(); - builder.Services.AddSingleton(new BatchedPullRequestActorId("repo", "branch")); + builder.Services.AddTransient(); + builder.Services.AddSingleton(new NonBatchedPullRequestUpdaterId(Guid.NewGuid())); + builder.Services.AddTransient(); + builder.Services.AddSingleton(new BatchedPullRequestUpdaterId("repo", "branch")); DependencyInjectionValidation.IsDependencyResolutionCoherent( s => diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs index 8fade6df40..abb60ab15e 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingCodeFlowUpdatesTests.cs @@ -8,7 +8,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture, NonParallelizable] -internal class PendingCodeFlowUpdatesTests : PendingUpdatePullRequestActorTests +internal class PendingCodeFlowUpdatesTests : PendingUpdatePullRequestUpdaterTests { [Test] public async Task PendingCodeFlowUpdatesNotUpdatablePr() diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestUpdaterTests.cs similarity index 81% rename from test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestUpdaterTests.cs index bd19a725dd..3cb9b4db3f 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatePullRequestUpdaterTests.cs @@ -6,15 +6,15 @@ namespace ProductConstructionService.DependencyFlow.Tests; -internal abstract class PendingUpdatePullRequestActorTests : PullRequestActorTests +internal abstract class PendingUpdatePullRequestUpdaterTests : PullRequestUpdaterTests { protected async Task WhenProcessPendingUpdatesAsyncIsCalled(Build forBuild, bool isCodeFlow = false) { await Execute( async context => { - IPullRequestActor actor = CreatePullRequestActor(context); - await actor.ProcessPendingUpdatesAsync(CreateSubscriptionUpdate(forBuild, isCodeFlow)); + IPullRequestUpdater updater = CreatePullRequestActor(context); + await updater.ProcessPendingUpdatesAsync(CreateSubscriptionUpdate(forBuild, isCodeFlow)); }); } diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs index fcdcacc735..f751b3f137 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PendingUpdatesTests.cs @@ -7,7 +7,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture, NonParallelizable] -internal class PendingUpdatesTests : PendingUpdatePullRequestActorTests +internal class PendingUpdatesTests : PendingUpdatePullRequestUpdaterTests { [Test] public async Task PendingUpdatesNotUpdatablePr() diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs index dc3c75818c..d6233199ec 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestBuilderTests.cs @@ -16,7 +16,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture] -internal class PullRequestBuilderTests : SubscriptionOrPullRequestActorTests +internal class PullRequestBuilderTests : SubscriptionOrPullRequestUpdaterTests { private Dictionary> _darcRemotes = null!; private Mock _remoteFactory = null!; diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs index 38cd93d29c..0273cef910 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestPolicyFailureNotifierTests.cs @@ -198,7 +198,7 @@ private InProgressPullRequest GetInProgressPullRequest(string url, int contained // the "Url" and ContainedSubscriptions fields in InProgressPullRequestObjects return new InProgressPullRequest() { - ActorId = new BatchedPullRequestActorId(FakeRepoName, "main").Id, + ActorId = new BatchedPullRequestUpdaterId(FakeRepoName, "main").Id, Url = url, ContainedSubscriptions = containedSubscriptions, SourceRepoNotified = false @@ -218,7 +218,7 @@ private InProgressPullRequest GetInProgressPullRequestWithoutTags(string url) return new InProgressPullRequest() { - ActorId = new BatchedPullRequestActorId(FakeRepoName, "main").Id, + ActorId = new BatchedPullRequestUpdaterId(FakeRepoName, "main").Id, Url = url, ContainedSubscriptions = containedSubscriptions, SourceRepoNotified = false diff --git a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs similarity index 97% rename from test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs index 0115a3b32f..a83702c383 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/PullRequestActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/PullRequestUpdaterTests.cs @@ -18,7 +18,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; -internal abstract class PullRequestActorTests : SubscriptionOrPullRequestActorTests +internal abstract class PullRequestUpdaterTests : SubscriptionOrPullRequestUpdaterTests { private const long InstallationId = 1174; protected const string InProgressPrUrl = "https://github.com/owner/repo/pull/10"; @@ -417,16 +417,16 @@ protected void ThenUpdateReminderIsRemoved() RemoveExpectedReminder(Subscription); } - protected IPullRequestActor CreatePullRequestActor(IServiceProvider context) + protected IPullRequestUpdater CreatePullRequestActor(IServiceProvider context) { - var actorFactory = context.GetRequiredService(); - return actorFactory.CreatePullRequestActor(GetPullRequestActorId()); + var updaterFactory = context.GetRequiredService(); + return updaterFactory.CreatePullRequestUpdater(GetPullRequestUpdaterId()); } protected SubscriptionUpdateWorkItem CreateSubscriptionUpdate(Build forBuild, bool isCodeFlow = false) => new() { - ActorId = GetPullRequestActorId().ToString(), + ActorId = GetPullRequestUpdaterId().ToString(), SubscriptionId = Subscription.Id, SubscriptionType = isCodeFlow ? SubscriptionType.DependenciesAndSources : SubscriptionType.Dependencies, BuildId = forBuild.Id, @@ -448,7 +448,7 @@ protected InProgressPullRequest CreatePullRequestCheckReminder( List? coherencyErrors = null) => new() { - ActorId = GetPullRequestActorId().ToString(), + ActorId = GetPullRequestUpdaterId().ToString(), ContainedSubscriptions = [ new SubscriptionPullRequestUpdate diff --git a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestUpdaterTests.cs similarity index 94% rename from test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestUpdaterTests.cs index be83d944ee..bf2cf4581f 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionOrPullRequestUpdaterTests.cs @@ -12,7 +12,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture] -internal abstract class SubscriptionOrPullRequestActorTests : ActorTests +internal abstract class SubscriptionOrPullRequestUpdaterTests : UpdaterTests { protected List> ContextUpdates = null!; protected List AfterDbUpdateActions = null!; @@ -158,10 +158,10 @@ internal Build GivenANewBuild(bool addToChannel, (string name, string version, b return build; } - protected PullRequestActorId GetPullRequestActorId() + protected PullRequestUpdaterId GetPullRequestUpdaterId() { return Subscription.PolicyObject.Batchable - ? new BatchedPullRequestActorId(Subscription.TargetRepository, Subscription.TargetBranch) - : new NonBatchedPullRequestActorId(Subscription.Id); + ? new BatchedPullRequestUpdaterId(Subscription.TargetRepository, Subscription.TargetBranch) + : new NonBatchedPullRequestUpdaterId(Subscription.Id); } } diff --git a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionUpdaterTests.cs similarity index 65% rename from test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/SubscriptionUpdaterTests.cs index 82f4b6d070..9c8cf4c7e2 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/SubscriptionUpdaterTests.cs @@ -12,33 +12,33 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture, NonParallelizable] -internal class SubscriptionActorTests : SubscriptionOrPullRequestActorTests +internal class SubscriptionUpdaterTests : SubscriptionOrPullRequestUpdaterTests { - protected Dictionary> PullRequestActors { get; private set; } = []; + protected Dictionary> PullRequestUpdaters { get; private set; } = []; [SetUp] - public void SubscriptionActorTests_SetUp() + public void SubscriptionUpdaterTests_SetUp() { - PullRequestActors.Clear(); + PullRequestUpdaters.Clear(); } protected override void RegisterServices(IServiceCollection services) { base.RegisterServices(services); - var actorFactory = new Mock(); + var updaterFactory = new Mock(); - actorFactory - .Setup(l => l.CreatePullRequestActor(It.IsAny())) - .Returns((PullRequestActorId actorId) => + updaterFactory + .Setup(l => l.CreatePullRequestUpdater(It.IsAny())) + .Returns((PullRequestUpdaterId updaterId) => { - Mock mock = PullRequestActors.GetOrAddValue( - actorId, - () => CreateMock()); + Mock mock = PullRequestUpdaters.GetOrAddValue( + updaterId, + () => CreateMock()); return mock.Object; }); - services.AddSingleton(actorFactory.Object); + services.AddSingleton(updaterFactory.Object); } internal async Task WhenUpdateAsyncIsCalled(Subscription forSubscription, Build andForBuild) @@ -46,15 +46,15 @@ internal async Task WhenUpdateAsyncIsCalled(Subscription forSubscription, Build await Execute( async provider => { - ISubscriptionActor actor = ActivatorUtilities.CreateInstance(provider, forSubscription.Id); - await actor.UpdateSubscriptionAsync(andForBuild.Id); + ISubscriptionTriggerer updater = ActivatorUtilities.CreateInstance(provider, forSubscription.Id); + await updater.UpdateSubscriptionAsync(andForBuild.Id); }); } - private void ThenUpdateAssetsAsyncShouldHaveBeenCalled(ActorId forActor, Build withBuild) + private void ThenUpdateAssetsAsyncShouldHaveBeenCalled(UpdaterId forUpdater, Build withBuild) { var updatedAssets = new List>(); - PullRequestActors.Should().ContainKey(forActor) + PullRequestUpdaters.Should().ContainKey(forUpdater) .WhoseValue.Verify( a => a.UpdateAssetsAsync(Subscription.Id, SubscriptionType.Dependencies, withBuild.Id, SourceRepo, NewCommit, Capture.In(updatedAssets))); @@ -85,7 +85,7 @@ public async Task BatchableEveryBuildSubscription() await WhenUpdateAsyncIsCalled(Subscription, b); ThenUpdateAssetsAsyncShouldHaveBeenCalled( - new BatchedPullRequestActorId(Subscription.TargetRepository, Subscription.TargetBranch), + new BatchedPullRequestUpdaterId(Subscription.TargetRepository, Subscription.TargetBranch), b); } @@ -102,6 +102,6 @@ public async Task NotBatchableEveryBuildSubscription() Build b = GivenANewBuild(true); await WhenUpdateAsyncIsCalled(Subscription, b); - ThenUpdateAssetsAsyncShouldHaveBeenCalled(new NonBatchedPullRequestActorId(Subscription.Id), b); + ThenUpdateAssetsAsyncShouldHaveBeenCalled(new NonBatchedPullRequestUpdaterId(Subscription.Id), b); } } diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs index da3a17227f..75d5c35ba1 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsForCodeFlowTests.cs @@ -12,7 +12,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; /// Each test should have the inner state that is left behind by the previous state. /// [TestFixture, NonParallelizable] -internal class UpdateAssetsForCodeFlowTests : UpdateAssetsPullRequestActorTests +internal class UpdateAssetsForCodeFlowTests : UpdateAssetsPullRequestUpdaterTests { [Test] public async Task UpdateWithNoExistingStateOrPrBranch() @@ -80,7 +80,7 @@ public async Task UpdateWithPrBranchReady() // For now, we manually update the info the unit tests expect. var expectedState = new WorkItems.InProgressPullRequest() { - ActorId = GetPullRequestActorId(Subscription).Id, + ActorId = GetPullRequestUpdaterId(Subscription).Id, Url = InProgressPrUrl, ContainedSubscriptions = [ diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestUpdaterTests.cs similarity index 81% rename from test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestUpdaterTests.cs index 75a0c0a4bc..1109631a63 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsPullRequestUpdaterTests.cs @@ -6,15 +6,15 @@ namespace ProductConstructionService.DependencyFlow.Tests; -internal abstract class UpdateAssetsPullRequestActorTests : PullRequestActorTests +internal abstract class UpdateAssetsPullRequestUpdaterTests : PullRequestUpdaterTests { protected async Task WhenUpdateAssetsAsyncIsCalled(Build forBuild) { await Execute( async context => { - IPullRequestActor actor = CreatePullRequestActor(context); - await actor.UpdateAssetsAsync( + IPullRequestUpdater updater = CreatePullRequestActor(context); + await updater.UpdateAssetsAsync( Subscription.Id, Subscription.SourceEnabled ? SubscriptionType.DependenciesAndSources : SubscriptionType.Dependencies, forBuild.Id, diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs index 0fd3e07857..cb675bd067 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdateAssetsTests.cs @@ -8,7 +8,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture, NonParallelizable] -internal class UpdateAssetsTests : UpdateAssetsPullRequestActorTests +internal class UpdateAssetsTests : UpdateAssetsPullRequestUpdaterTests { [TestCase(false)] [TestCase(true)] diff --git a/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs similarity index 80% rename from test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs rename to test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs index 7c3a3cdb72..d9ccfa64eb 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/ActorTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs @@ -20,7 +20,7 @@ namespace ProductConstructionService.DependencyFlow.Tests; [TestFixture] -internal abstract class ActorTests : TestsWithServices +internal abstract class UpdaterTests : TestsWithServices { protected const string AssetFeedUrl = "https://source.feed/index.json"; protected const string SourceBranch = "source.branch"; @@ -30,11 +30,11 @@ internal abstract class ActorTests : TestsWithServices protected const string NewBuildNumber = "build.number"; protected const string NewCommit = "sha2"; - protected Dictionary ExpectedActorState { get; private set; } = null!; + protected Dictionary ExpectedCacheState { get; private set; } = null!; protected Dictionary ExpectedReminders { get; private set; } = null!; protected MockReminderManagerFactory Reminders { get; private set; } = null!; - protected MockRedisCacheFactory RedisCache { get; private set; } = null!; + protected MockRedisCacheFactory Cache { get; private set; } = null!; protected Mock RemoteFactory { get; private set; } = null!; protected Dictionary> DarcRemotes { get; private set; } = null!; @@ -48,7 +48,7 @@ protected override void RegisterServices(IServiceCollection services) { base.RegisterServices(services); services.AddDependencyFlowProcessors(); - services.AddSingleton(RedisCache); + services.AddSingleton(Cache); services.AddSingleton(Reminders); services.AddOperationTracking(_ => { }); services.AddSingleton(); @@ -77,11 +77,11 @@ protected override void RegisterServices(IServiceCollection services) } [SetUp] - public void ActorTests_SetUp() + public void UpdaterTests_SetUp() { - ExpectedActorState = []; + ExpectedCacheState = []; ExpectedReminders = []; - RedisCache = new(); + Cache = new(); Reminders = new(); RemoteFactory = new(); DarcRemotes = new() @@ -94,48 +94,48 @@ public void ActorTests_SetUp() } [TearDown] - public void ActorTests_TearDown() + public void UpdaterTests_TearDown() { var ExcludeWorkItemId = static (FluentAssertions.Equivalency.EquivalencyAssertionOptions> opt) => opt.Excluding(member => member.DeclaringType == typeof(WorkItem) && member.Name.Equals(nameof(WorkItem.Id))); - RedisCache.Data.Should().BeEquivalentTo(ExpectedActorState, ExcludeWorkItemId); + Cache.Data.Should().BeEquivalentTo(ExpectedCacheState, ExcludeWorkItemId); Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders, ExcludeWorkItemId); } protected void SetState(Subscription subscription, T state) where T : class { - RedisCache.Data[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = state; + Cache.Data[typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)] = state; } protected void RemoveState(Subscription subscription) where T : class { - RedisCache.Data.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + Cache.Data.Remove(typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)); } protected void SetExpectedReminder(Subscription subscription, T reminder) where T : WorkItem { - ExpectedReminders[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = reminder; + ExpectedReminders[typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)] = reminder; } protected void RemoveExpectedReminder(Subscription subscription) where T : WorkItem { - ExpectedReminders.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + ExpectedReminders.Remove(typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)); } protected void SetExpectedState(Subscription subscription, T state) where T : class { - ExpectedActorState[typeof(T).Name + "_" + GetPullRequestActorId(subscription)] = state; + ExpectedCacheState[typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)] = state; } protected void RemoveExpectedState(Subscription subscription) where T : class { - ExpectedActorState.Remove(typeof(T).Name + "_" + GetPullRequestActorId(subscription)); + ExpectedCacheState.Remove(typeof(T).Name + "_" + GetPullRequestUpdaterId(subscription)); } - protected static PullRequestActorId GetPullRequestActorId(Subscription subscription) + protected static PullRequestUpdaterId GetPullRequestUpdaterId(Subscription subscription) { return subscription.PolicyObject.Batchable - ? new BatchedPullRequestActorId(subscription.TargetRepository, subscription.TargetBranch) - : new NonBatchedPullRequestActorId(subscription.Id); + ? new BatchedPullRequestUpdaterId(subscription.TargetRepository, subscription.TargetBranch) + : new NonBatchedPullRequestUpdaterId(subscription.Id); } } From 638c7948303ad12d500d0e16ac8bb832f3d8eda8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 29 Aug 2024 12:29:19 +0200 Subject: [PATCH 12/33] Re-generate the PCS client based on newest API (#3891) --- src/Maestro/Client/src/MaestroApiFactory.cs | 2 +- src/Maestro/Client/src/MaestroApiOptions.cs | 27 +- .../SubscriptionActorService/Program.cs | 2 +- .../PullRequestActor.cs | 16 +- .../Darc/Helpers/LocalSettings.cs | 4 +- .../Generated/Assets.cs | 598 ++++++++++++++ .../Generated/BuildTime.cs | 111 +++ .../Generated/Builds.cs | 772 ++++++++++++++++++ .../Generated/Channels.cs | 680 +++++++++++++++ .../Generated/CodeFlow.cs | 22 +- .../Generated/DefaultChannels.cs | 435 ++++++++++ .../Generated/Goal.cs | 208 +++++ .../Generated/Models/ApiError.cs | 21 + .../Generated/Models/Asset.cs | 39 + .../Generated/Models/AssetAndLocation.cs | 38 + .../Generated/Models/AssetData.cs | 28 + .../Generated/Models/AssetLocation.cs | 39 + .../Generated/Models/AssetLocationData.cs | 34 + .../Generated/Models/Build.cs | 83 ++ .../Generated/Models/BuildData.cs | 101 +++ .../Generated/Models/BuildGraph.cs | 32 + .../Generated/Models/BuildIncoherence.cs | 26 + .../Generated/Models/BuildRef.cs | 26 + .../Generated/Models/BuildTime.cs | 26 + .../Generated/Models/BuildUpdate.cs | 17 + .../Generated/Models/Channel.cs | 43 + .../Generated/Models/CodeFlowRequest.cs | 8 +- .../Generated/Models/Commit.cs | 23 + .../Generated/Models/DefaultChannel.cs | 45 + .../Models/DefaultChannelCreateData.cs | 46 ++ .../Models/DefaultChannelUpdateData.cs | 26 + .../Generated/Models/FlowEdge.cs | 45 + .../Generated/Models/FlowGraph.cs | 40 + .../Generated/Models/FlowRef.cs | 54 ++ .../Generated/Models/Goal.cs | 25 + .../Generated/Models/GoalRequestJson.cs | 18 + .../Generated/Models/LocationType.cs | 17 + .../Generated/Models/MergePolicy.cs | 21 + .../Generated/Models/ReleasePipeline.cs | 28 + .../Generated/Models/RepositoryBranch.cs | 24 + .../Generated/Models/RepositoryHistoryItem.cs | 43 + .../Generated/Models/Subscription.cs | 65 ++ .../Generated/Models/SubscriptionData.cs | 82 ++ .../Models/SubscriptionHistoryItem.cs | 39 + .../Generated/Models/SubscriptionPolicy.cs | 39 + .../Generated/Models/SubscriptionUpdate.cs | 42 + .../Generated/Models/UpdateFrequency.cs | 21 + .../Generated/Pipelines.cs | 373 +++++++++ .../ProductConstructionServiceApi.cs | 42 +- .../Generated/Repository.cs | 465 +++++++++++ .../Generated/Status.cs | 208 ----- .../Generated/Subscriptions.cs | 752 +++++++++++++++++ .../PcsApiFactory.cs | 15 +- .../ProductConstructionService.Client.csproj | 2 +- .../ProductConstructionServiceApiOptions.cs | 36 +- .../VmrTestsBase.cs | 2 +- .../PullRequestActorTests.cs | 12 +- 57 files changed, 5818 insertions(+), 270 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Assets.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/BuildTime.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Builds.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Channels.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/DefaultChannels.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Goal.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ApiError.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Asset.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetAndLocation.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocation.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocationData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Build.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildGraph.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildIncoherence.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildRef.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildTime.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildUpdate.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Channel.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Commit.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannel.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelCreateData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelUpdateData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowEdge.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowGraph.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowRef.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Goal.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/GoalRequestJson.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/LocationType.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/MergePolicy.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ReleasePipeline.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryBranch.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryHistoryItem.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Subscription.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionData.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionHistoryItem.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionPolicy.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionUpdate.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/UpdateFrequency.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Pipelines.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Repository.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Status.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Client/Generated/Subscriptions.cs diff --git a/src/Maestro/Client/src/MaestroApiFactory.cs b/src/Maestro/Client/src/MaestroApiFactory.cs index bc60def6d6..a2cf94ffd5 100644 --- a/src/Maestro/Client/src/MaestroApiFactory.cs +++ b/src/Maestro/Client/src/MaestroApiFactory.cs @@ -40,7 +40,7 @@ public static IMaestroApi GetAuthenticated( bool disableInteractiveAuth) { return new MaestroApi(new MaestroApiOptions( - MaestroApiOptions.StagingBuildAssetRegistryBaseUri, + MaestroApiOptions.StagingMaestroUri, accessToken, managedIdentityId, disableInteractiveAuth)); diff --git a/src/Maestro/Client/src/MaestroApiOptions.cs b/src/Maestro/Client/src/MaestroApiOptions.cs index bba86e5697..d6d549c4ba 100644 --- a/src/Maestro/Client/src/MaestroApiOptions.cs +++ b/src/Maestro/Client/src/MaestroApiOptions.cs @@ -11,26 +11,31 @@ namespace Microsoft.DotNet.Maestro.Client { public partial class MaestroApiOptions { - public const string ProductionBuildAssetRegistryBaseUri = "https://maestro.dot.net/"; - public const string OldProductionBuildAssetRegistryBaseUri = "https://maestro-prod.westus2.cloudapp.azure.com/"; - // https://ms.portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/54c17f3d-7325-4eca-9db7-f090bfc765a8/isMSAApp~/false private const string MaestroProductionAppId = "54c17f3d-7325-4eca-9db7-f090bfc765a8"; - public const string StagingBuildAssetRegistryBaseUri = "https://maestro.int-dot.net/"; - public const string OldStagingBuildAssetRegistryBaseUri = "https://maestro-int.westus2.cloudapp.azure.com/"; - // https://ms.portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/baf98f1b-374e-487d-af42-aa33807f11e4/isMSAApp~/false private const string MaestroStagingAppId = "baf98f1b-374e-487d-af42-aa33807f11e4"; + public const string ProductionMaestroUri = "https://maestro.dot.net/"; + public const string OldProductionMaestroUri = "https://maestro-prod.westus2.cloudapp.azure.com/"; + + public const string StagingMaestroUri = "https://maestro.int-dot.net/"; + public const string OldPcsStagingUri = "https://maestro-int.westus2.cloudapp.azure.com/"; + public const string PcsStagingUri = "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; + public const string PcsLocalUri = "https://localhost:53180/"; + private const string APP_USER_SCOPE = "Maestro.User"; private static readonly Dictionary EntraAppIds = new Dictionary { - [StagingBuildAssetRegistryBaseUri.TrimEnd('/')] = MaestroStagingAppId, - [OldStagingBuildAssetRegistryBaseUri.TrimEnd('/')] = MaestroStagingAppId, - [ProductionBuildAssetRegistryBaseUri.TrimEnd('/')] = MaestroProductionAppId, - [OldProductionBuildAssetRegistryBaseUri.TrimEnd('/')] = MaestroProductionAppId, + [StagingMaestroUri.TrimEnd('/')] = MaestroStagingAppId, + [OldPcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, + [PcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, + [PcsLocalUri.TrimEnd('/')] = MaestroStagingAppId, + + [ProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, + [OldProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, }; /// @@ -44,7 +49,7 @@ public MaestroApiOptions(string baseUri, string accessToken, string managedIdent : this( new Uri(baseUri), AppCredentialResolver.CreateCredential( - new AppCredentialResolverOptions(EntraAppIds[(baseUri ?? ProductionBuildAssetRegistryBaseUri).TrimEnd('/')]) + new AppCredentialResolverOptions(EntraAppIds[(baseUri ?? ProductionMaestroUri).TrimEnd('/')]) { DisableInteractiveAuth = disableInteractiveAuth, Token = accessToken, diff --git a/src/Maestro/SubscriptionActorService/Program.cs b/src/Maestro/SubscriptionActorService/Program.cs index 9a92d6a496..6c2c1ec5ff 100644 --- a/src/Maestro/SubscriptionActorService/Program.cs +++ b/src/Maestro/SubscriptionActorService/Program.cs @@ -85,7 +85,7 @@ public static void Configure(IServiceCollection services) return PcsApiFactory.GetAnonymous(uri); } - return PcsApiFactory.GetAuthenticated(uri, managedIdentityId: "system"); + return PcsApiFactory.GetAuthenticated(uri, managedIdentityId: "system", disableInteractiveAuth: true); }); services.AddMergePolicies(); diff --git a/src/Maestro/SubscriptionActorService/PullRequestActor.cs b/src/Maestro/SubscriptionActorService/PullRequestActor.cs index 39b0d71ac4..179bd1b06e 100644 --- a/src/Maestro/SubscriptionActorService/PullRequestActor.cs +++ b/src/Maestro/SubscriptionActorService/PullRequestActor.cs @@ -1040,7 +1040,7 @@ private async Task GetRepositoryBranchUpdate() RepositoryBranchUpdate? update = await _context.RepositoryBranchUpdates.FindAsync(repo, branch); if (update == null) { - RepositoryBranch repoBranch = await GetRepositoryBranch(repo, branch); + var repoBranch = await GetRepositoryBranch(repo, branch); _context.RepositoryBranchUpdates.Add( update = new RepositoryBranchUpdate { RepositoryBranch = repoBranch }); } @@ -1052,13 +1052,13 @@ private async Task GetRepositoryBranchUpdate() return update; } - private async Task GetRepositoryBranch(string repo, string branch) + private async Task GetRepositoryBranch(string repo, string branch) { - RepositoryBranch? repoBranch = await _context.RepositoryBranches.FindAsync(repo, branch); + var repoBranch = await _context.RepositoryBranches.FindAsync(repo, branch); if (repoBranch == null) { _context.RepositoryBranches.Add( - repoBranch = new RepositoryBranch + repoBranch = new Maestro.Data.Models.RepositoryBranch { RepositoryName = repo, BranchName = branch @@ -1160,10 +1160,8 @@ private async Task> ProcessCodeFlowUpdatesAsync( try { - await _pcsClient.CodeFlow.FlowAsync(new CodeFlowRequest + await _pcsClient.CodeFlow.FlowBuildAsync(new CodeFlowRequest(update.SubscriptionId, update.BuildId) { - BuildId = update.BuildId, - SubscriptionId = update.SubscriptionId, PrBranch = codeFlowStatus.PrBranch, PrUrl = pr.Url, }); @@ -1203,10 +1201,8 @@ private async Task> RequestCodeFlowBranchAsync(UpdateAssetsPa try { - await _pcsClient.CodeFlow.FlowAsync(new CodeFlowRequest + await _pcsClient.CodeFlow.FlowBuildAsync(new CodeFlowRequest(update.SubscriptionId, update.BuildId) { - BuildId = update.BuildId, - SubscriptionId = update.SubscriptionId, PrBranch = codeFlowUpdate.PrBranch, }); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs b/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs index b80f68c855..03c67343a7 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Helpers/LocalSettings.cs @@ -23,7 +23,7 @@ internal class LocalSettings public string AzureDevOpsToken { get; set; } - public string BuildAssetRegistryBaseUri { get; set; } = MaestroApiOptions.ProductionBuildAssetRegistryBaseUri; + public string BuildAssetRegistryBaseUri { get; set; } = MaestroApiOptions.ProductionMaestroUri; /// /// Saves the settings in the settings files @@ -82,7 +82,7 @@ static string PreferOptionToSetting(string option, string localSetting) localSettings.BuildAssetRegistryToken = PreferOptionToSetting(options.BuildAssetRegistryToken, localSettings.BuildAssetRegistryToken); localSettings.BuildAssetRegistryBaseUri = options.BuildAssetRegistryBaseUri ?? localSettings.BuildAssetRegistryBaseUri - ?? MaestroApiOptions.ProductionBuildAssetRegistryBaseUri; + ?? MaestroApiOptions.ProductionMaestroUri; return localSettings; } diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Assets.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Assets.cs new file mode 100644 index 0000000000..27b2e13252 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Assets.cs @@ -0,0 +1,598 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IAssets + { + Task BulkAddLocationsAsync( + IImmutableList body, + CancellationToken cancellationToken = default + ); + + AsyncPageable ListAssetsAsync( + int? buildId = default, + bool? loadLocations = default, + string name = default, + bool? nonShipping = default, + string version = default, + CancellationToken cancellationToken = default + ); + + Task> ListAssetsPageAsync( + int? buildId = default, + bool? loadLocations = default, + string name = default, + bool? nonShipping = default, + int? page = default, + int? perPage = default, + string version = default, + CancellationToken cancellationToken = default + ); + + Task GetDarcVersionAsync( + CancellationToken cancellationToken = default + ); + + Task GetAssetAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task AddAssetLocationToAssetAsync( + int assetId, + Models.LocationType assetLocationType, + string location, + CancellationToken cancellationToken = default + ); + + Task RemoveAssetLocationFromAssetAsync( + int assetId, + int assetLocationId, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Assets : IServiceOperations, IAssets + { + public Assets(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedBulkAddLocationsRequest(RestApiException ex); + + public async Task BulkAddLocationsAsync( + IImmutableList body, + CancellationToken cancellationToken = default + ) + { + + if (body == default(IImmutableList)) + { + throw new ArgumentNullException(nameof(body)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets/bulk-add-locations", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + if (body != default(IImmutableList)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnBulkAddLocationsFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnBulkAddLocationsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedBulkAddLocationsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedListAssetsRequest(RestApiException ex); + + public AsyncPageable ListAssetsAsync( + int? buildId = default, + bool? loadLocations = default, + string name = default, + bool? nonShipping = default, + string version = default, + CancellationToken cancellationToken = default + ) + { + async IAsyncEnumerable> GetPages(string _continueToken, int? _pageSizeHint) + { + int? page = 1; + int? perPage = _pageSizeHint; + + if (!string.IsNullOrEmpty(_continueToken)) + { + page = int.Parse(_continueToken); + } + + while (true) + { + Page _page = null; + + try { + _page = await ListAssetsPageAsync( + buildId, + loadLocations, + name, + nonShipping, + page, + perPage, + version, + cancellationToken + ).ConfigureAwait(false); + if (_page.Values.Count < 1) + { + yield break; + } + } + catch (RestApiException e) when (e.Response.Status == 404) + { + yield break; + } + + yield return _page; + page++; + } + } + return AsyncPageable.Create(GetPages); + } + + public async Task> ListAssetsPageAsync( + int? buildId = default, + bool? loadLocations = default, + string name = default, + bool? nonShipping = default, + int? page = default, + int? perPage = default, + string version = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets", + false); + + if (!string.IsNullOrEmpty(name)) + { + _url.AppendQuery("name", Client.Serialize(name)); + } + if (!string.IsNullOrEmpty(version)) + { + _url.AppendQuery("version", Client.Serialize(version)); + } + if (buildId != default) + { + _url.AppendQuery("buildId", Client.Serialize(buildId)); + } + if (nonShipping != default) + { + _url.AppendQuery("nonShipping", Client.Serialize(nonShipping)); + } + if (loadLocations != default) + { + _url.AppendQuery("loadLocations", Client.Serialize(loadLocations)); + } + if (page != default) + { + _url.AppendQuery("page", Client.Serialize(page)); + } + if (perPage != default) + { + _url.AppendQuery("perPage", Client.Serialize(perPage)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListAssetsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListAssetsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return Page.FromValues(_body, (page + 1).ToString(), _res); + } + } + } + } + + internal async Task OnListAssetsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListAssetsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetDarcVersionRequest(RestApiException ex); + + public async Task GetDarcVersionAsync( + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets/darc-version", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetDarcVersionFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetDarcVersionFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetDarcVersionFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetDarcVersionRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetAssetRequest(RestApiException ex); + + public async Task GetAssetAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetAssetFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetAssetFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetAssetFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetAssetRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedAddAssetLocationToAssetRequest(RestApiException ex); + + public async Task AddAssetLocationToAssetAsync( + int assetId, + Models.LocationType assetLocationType, + string location, + CancellationToken cancellationToken = default + ) + { + + if (assetLocationType == default) + { + throw new ArgumentNullException(nameof(assetLocationType)); + } + + if (string.IsNullOrEmpty(location)) + { + throw new ArgumentNullException(nameof(location)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets/{assetId}/locations".Replace("{assetId}", Uri.EscapeDataString(Client.Serialize(assetId))), + false); + + if (!string.IsNullOrEmpty(location)) + { + _url.AppendQuery("location", Client.Serialize(location)); + } + if (assetLocationType != default) + { + _url.AppendQuery("assetLocationType", Client.Serialize(assetLocationType)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnAddAssetLocationToAssetFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnAddAssetLocationToAssetFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnAddAssetLocationToAssetFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedAddAssetLocationToAssetRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedRemoveAssetLocationFromAssetRequest(RestApiException ex); + + public async Task RemoveAssetLocationFromAssetAsync( + int assetId, + int assetLocationId, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/assets/{assetId}/locations/{assetLocationId}".Replace("{assetId}", Uri.EscapeDataString(Client.Serialize(assetId))).Replace("{assetLocationId}", Uri.EscapeDataString(Client.Serialize(assetLocationId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnRemoveAssetLocationFromAssetFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnRemoveAssetLocationFromAssetFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedRemoveAssetLocationFromAssetRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/BuildTime.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/BuildTime.cs new file mode 100644 index 0000000000..fc8c050538 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/BuildTime.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IBuildTime + { + Task GetBuildTimesAsync( + int days, + int id, + CancellationToken cancellationToken = default + ); + + } + + internal partial class BuildTime : IServiceOperations, IBuildTime + { + public BuildTime(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedGetBuildTimesRequest(RestApiException ex); + + public async Task GetBuildTimesAsync( + int days, + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/buildtime/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + if (days != default) + { + _url.AppendQuery("days", Client.Serialize(days)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetBuildTimesFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetBuildTimesFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetBuildTimesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetBuildTimesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Builds.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Builds.cs new file mode 100644 index 0000000000..3718298afd --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Builds.cs @@ -0,0 +1,772 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IBuilds + { + AsyncPageable ListBuildsAsync( + string azdoAccount = default, + int? azdoBuildId = default, + string azdoProject = default, + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + string repository = default, + CancellationToken cancellationToken = default + ); + + Task> ListBuildsPageAsync( + string azdoAccount = default, + int? azdoBuildId = default, + string azdoProject = default, + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + int? page = default, + int? perPage = default, + string repository = default, + CancellationToken cancellationToken = default + ); + + Task CreateAsync( + Models.BuildData body, + CancellationToken cancellationToken = default + ); + + Task GetBuildAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task GetBuildGraphAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task GetLatestAsync( + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + string repository = default, + CancellationToken cancellationToken = default + ); + + Task GetCommitAsync( + int buildId, + CancellationToken cancellationToken = default + ); + + Task UpdateAsync( + Models.BuildUpdate body, + int buildId, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Builds : IServiceOperations, IBuilds + { + public Builds(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListBuildsRequest(RestApiException ex); + + public AsyncPageable ListBuildsAsync( + string azdoAccount = default, + int? azdoBuildId = default, + string azdoProject = default, + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + string repository = default, + CancellationToken cancellationToken = default + ) + { + async IAsyncEnumerable> GetPages(string _continueToken, int? _pageSizeHint) + { + int? page = 1; + int? perPage = _pageSizeHint; + + if (!string.IsNullOrEmpty(_continueToken)) + { + page = int.Parse(_continueToken); + } + + while (true) + { + Page _page = null; + + try { + _page = await ListBuildsPageAsync( + azdoAccount, + azdoBuildId, + azdoProject, + buildNumber, + commit, + channelId, + loadCollections, + notAfter, + notBefore, + page, + perPage, + repository, + cancellationToken + ).ConfigureAwait(false); + if (_page.Values.Count < 1) + { + yield break; + } + } + catch (RestApiException e) when (e.Response.Status == 404) + { + yield break; + } + + yield return _page; + page++; + } + } + return AsyncPageable.Create(GetPages); + } + + public async Task> ListBuildsPageAsync( + string azdoAccount = default, + int? azdoBuildId = default, + string azdoProject = default, + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + int? page = default, + int? perPage = default, + string repository = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(commit)) + { + _url.AppendQuery("commit", Client.Serialize(commit)); + } + if (!string.IsNullOrEmpty(buildNumber)) + { + _url.AppendQuery("buildNumber", Client.Serialize(buildNumber)); + } + if (azdoBuildId != default) + { + _url.AppendQuery("azdoBuildId", Client.Serialize(azdoBuildId)); + } + if (!string.IsNullOrEmpty(azdoAccount)) + { + _url.AppendQuery("azdoAccount", Client.Serialize(azdoAccount)); + } + if (!string.IsNullOrEmpty(azdoProject)) + { + _url.AppendQuery("azdoProject", Client.Serialize(azdoProject)); + } + if (channelId != default) + { + _url.AppendQuery("channelId", Client.Serialize(channelId)); + } + if (notBefore != default) + { + _url.AppendQuery("notBefore", Client.Serialize(notBefore)); + } + if (notAfter != default) + { + _url.AppendQuery("notAfter", Client.Serialize(notAfter)); + } + if (loadCollections != default) + { + _url.AppendQuery("loadCollections", Client.Serialize(loadCollections)); + } + if (page != default) + { + _url.AppendQuery("page", Client.Serialize(page)); + } + if (perPage != default) + { + _url.AppendQuery("perPage", Client.Serialize(perPage)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListBuildsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListBuildsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return Page.FromValues(_body, (page + 1).ToString(), _res); + } + } + } + } + + internal async Task OnListBuildsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListBuildsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedCreateRequest(RestApiException ex); + + public async Task CreateAsync( + Models.BuildData body, + CancellationToken cancellationToken = default + ) + { + + if (body == default(Models.BuildData)) + { + throw new ArgumentNullException(nameof(body)); + } + + if (!body.IsValid) + { + throw new ArgumentException("The parameter is not valid", nameof(body)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + if (body != default(Models.BuildData)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnCreateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetBuildRequest(RestApiException ex); + + public async Task GetBuildAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetBuildFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetBuildFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetBuildFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetBuildRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetBuildGraphRequest(RestApiException ex); + + public async Task GetBuildGraphAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds/{id}/graph".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetBuildGraphFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetBuildGraphFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetBuildGraphFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetBuildGraphRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetLatestRequest(RestApiException ex); + + public async Task GetLatestAsync( + string buildNumber = default, + string commit = default, + int? channelId = default, + bool? loadCollections = default, + DateTimeOffset? notAfter = default, + DateTimeOffset? notBefore = default, + string repository = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds/latest", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(commit)) + { + _url.AppendQuery("commit", Client.Serialize(commit)); + } + if (!string.IsNullOrEmpty(buildNumber)) + { + _url.AppendQuery("buildNumber", Client.Serialize(buildNumber)); + } + if (channelId != default) + { + _url.AppendQuery("channelId", Client.Serialize(channelId)); + } + if (notBefore != default) + { + _url.AppendQuery("notBefore", Client.Serialize(notBefore)); + } + if (notAfter != default) + { + _url.AppendQuery("notAfter", Client.Serialize(notAfter)); + } + if (loadCollections != default) + { + _url.AppendQuery("loadCollections", Client.Serialize(loadCollections)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetLatestFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetLatestFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetLatestFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetLatestRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetCommitRequest(RestApiException ex); + + public async Task GetCommitAsync( + int buildId, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds/{buildId}/commit".Replace("{buildId}", Uri.EscapeDataString(Client.Serialize(buildId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetCommitFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetCommitFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetCommitFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetCommitRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedUpdateRequest(RestApiException ex); + + public async Task UpdateAsync( + Models.BuildUpdate body, + int buildId, + CancellationToken cancellationToken = default + ) + { + + if (body == default(Models.BuildUpdate)) + { + throw new ArgumentNullException(nameof(body)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/builds/{buildId}".Replace("{buildId}", Uri.EscapeDataString(Client.Serialize(buildId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Patch; + + if (body != default(Models.BuildUpdate)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnUpdateFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnUpdateFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnUpdateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedUpdateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Channels.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Channels.cs new file mode 100644 index 0000000000..bc36f30ac0 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Channels.cs @@ -0,0 +1,680 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IChannels + { + Task> ListChannelsAsync( + string classification = default, + CancellationToken cancellationToken = default + ); + + Task CreateChannelAsync( + string classification, + string name, + CancellationToken cancellationToken = default + ); + + Task> ListRepositoriesAsync( + int id, + int? withBuildsInDays = default, + CancellationToken cancellationToken = default + ); + + Task GetChannelAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task DeleteChannelAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task AddBuildToChannelAsync( + int buildId, + int channelId, + CancellationToken cancellationToken = default + ); + + Task RemoveBuildFromChannelAsync( + int buildId, + int channelId, + CancellationToken cancellationToken = default + ); + + Task GetFlowGraphAsync( + int days, + int channelId, + bool includeArcade, + bool includeBuildTimes, + bool includeDisabledSubscriptions, + IImmutableList includedFrequencies = default, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Channels : IServiceOperations, IChannels + { + public Channels(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListChannelsRequest(RestApiException ex); + + public async Task> ListChannelsAsync( + string classification = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels", + false); + + if (!string.IsNullOrEmpty(classification)) + { + _url.AppendQuery("classification", Client.Serialize(classification)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListChannelsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListChannelsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListChannelsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListChannelsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedCreateChannelRequest(RestApiException ex); + + public async Task CreateChannelAsync( + string classification, + string name, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(classification)) + { + throw new ArgumentNullException(nameof(classification)); + } + + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels", + false); + + if (!string.IsNullOrEmpty(name)) + { + _url.AppendQuery("name", Client.Serialize(name)); + } + if (!string.IsNullOrEmpty(classification)) + { + _url.AppendQuery("classification", Client.Serialize(classification)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreateChannelFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnCreateChannelFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnCreateChannelFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreateChannelRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedListRepositoriesRequest(RestApiException ex); + + public async Task> ListRepositoriesAsync( + int id, + int? withBuildsInDays = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{id}/repositories".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + if (withBuildsInDays != default) + { + _url.AppendQuery("withBuildsInDays", Client.Serialize(withBuildsInDays)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListRepositoriesFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListRepositoriesFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListRepositoriesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListRepositoriesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetChannelRequest(RestApiException ex); + + public async Task GetChannelAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetChannelFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetChannelFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetChannelFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetChannelRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedDeleteChannelRequest(RestApiException ex); + + public async Task DeleteChannelAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnDeleteChannelFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnDeleteChannelFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnDeleteChannelFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedDeleteChannelRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedAddBuildToChannelRequest(RestApiException ex); + + public async Task AddBuildToChannelAsync( + int buildId, + int channelId, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{channelId}/builds/{buildId}".Replace("{channelId}", Uri.EscapeDataString(Client.Serialize(channelId))).Replace("{buildId}", Uri.EscapeDataString(Client.Serialize(buildId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnAddBuildToChannelFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnAddBuildToChannelFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedAddBuildToChannelRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedRemoveBuildFromChannelRequest(RestApiException ex); + + public async Task RemoveBuildFromChannelAsync( + int buildId, + int channelId, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{channelId}/builds/{buildId}".Replace("{channelId}", Uri.EscapeDataString(Client.Serialize(channelId))).Replace("{buildId}", Uri.EscapeDataString(Client.Serialize(buildId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnRemoveBuildFromChannelFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnRemoveBuildFromChannelFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedRemoveBuildFromChannelRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetFlowGraphRequest(RestApiException ex); + + public async Task GetFlowGraphAsync( + int days, + int channelId, + bool includeArcade, + bool includeBuildTimes, + bool includeDisabledSubscriptions, + IImmutableList includedFrequencies = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/channels/{channelId}/graph".Replace("{channelId}", Uri.EscapeDataString(Client.Serialize(channelId))), + false); + + if (includeDisabledSubscriptions != default) + { + _url.AppendQuery("includeDisabledSubscriptions", Client.Serialize(includeDisabledSubscriptions)); + } + if (includedFrequencies != default(IImmutableList)) + { + foreach (var _item in includedFrequencies) + { + _url.AppendQuery("includedFrequencies", Client.Serialize(_item)); + } + } + if (includeBuildTimes != default) + { + _url.AppendQuery("includeBuildTimes", Client.Serialize(includeBuildTimes)); + } + if (days != default) + { + _url.AppendQuery("days", Client.Serialize(days)); + } + if (includeArcade != default) + { + _url.AppendQuery("includeArcade", Client.Serialize(includeArcade)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetFlowGraphFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetFlowGraphFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetFlowGraphFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetFlowGraphRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/CodeFlow.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/CodeFlow.cs index 4f4fdd48d6..18c5041309 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/Generated/CodeFlow.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/CodeFlow.cs @@ -15,7 +15,7 @@ namespace ProductConstructionService.Client { public partial interface ICodeFlow { - Task FlowAsync( + Task FlowBuildAsync( Models.CodeFlowRequest body, CancellationToken cancellationToken = default ); @@ -33,9 +33,9 @@ public CodeFlow(ProductConstructionServiceApi client) partial void HandleFailedRequest(RestApiException ex); - partial void HandleFailedFlowRequest(RestApiException ex); + partial void HandleFailedFlowBuildRequest(RestApiException ex); - public async Task FlowAsync( + public async Task FlowBuildAsync( Models.CodeFlowRequest body, CancellationToken cancellationToken = default ) @@ -46,14 +46,16 @@ public async Task FlowAsync( throw new ArgumentNullException(nameof(body)); } + const string apiVersion = "2020-02-20"; var _baseUri = Client.Options.BaseUri; var _url = new RequestUriBuilder(); _url.Reset(_baseUri); _url.AppendPath( - "/codeflow", + "/api/codeflow", false); + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); using (var _req = Client.Pipeline.CreateRequest()) @@ -71,7 +73,7 @@ public async Task FlowAsync( { if (_res.Status < 200 || _res.Status >= 300) { - await OnFlowFailed(_req, _res).ConfigureAwait(false); + await OnFlowBuildFailed(_req, _res).ConfigureAwait(false); } @@ -80,7 +82,7 @@ public async Task FlowAsync( } } - internal async Task OnFlowFailed(Request req, Response res) + internal async Task OnFlowBuildFailed(Request req, Response res) { string content = null; if (res.ContentStream != null) @@ -91,11 +93,13 @@ internal async Task OnFlowFailed(Request req, Response res) } } - var ex = new RestApiException( + var ex = new RestApiException( req, res, - content); - HandleFailedFlowRequest(ex); + content, + Client.Deserialize(content) + ); + HandleFailedFlowBuildRequest(ex); HandleFailedRequest(ex); Client.OnFailedRequest(ex); throw ex; diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/DefaultChannels.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/DefaultChannels.cs new file mode 100644 index 0000000000..82fd112125 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/DefaultChannels.cs @@ -0,0 +1,435 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IDefaultChannels + { + Task> ListAsync( + string branch = default, + bool? enabled = default, + int? channelId = default, + string repository = default, + CancellationToken cancellationToken = default + ); + + Task CreateAsync( + Models.DefaultChannelCreateData body, + CancellationToken cancellationToken = default + ); + + Task UpdateAsync( + int id, + Models.DefaultChannelUpdateData body = default, + CancellationToken cancellationToken = default + ); + + Task GetAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task DeleteAsync( + int id, + CancellationToken cancellationToken = default + ); + + } + + internal partial class DefaultChannels : IServiceOperations, IDefaultChannels + { + public DefaultChannels(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListRequest(RestApiException ex); + + public async Task> ListAsync( + string branch = default, + bool? enabled = default, + int? channelId = default, + string repository = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/default-channels", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("branch", Client.Serialize(branch)); + } + if (channelId != default) + { + _url.AppendQuery("channelId", Client.Serialize(channelId)); + } + if (enabled != default) + { + _url.AppendQuery("enabled", Client.Serialize(enabled)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedCreateRequest(RestApiException ex); + + public async Task CreateAsync( + Models.DefaultChannelCreateData body, + CancellationToken cancellationToken = default + ) + { + + if (body == default(Models.DefaultChannelCreateData)) + { + throw new ArgumentNullException(nameof(body)); + } + + if (!body.IsValid) + { + throw new ArgumentException("The parameter is not valid", nameof(body)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/default-channels", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + if (body != default(Models.DefaultChannelCreateData)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnCreateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedUpdateRequest(RestApiException ex); + + public async Task UpdateAsync( + int id, + Models.DefaultChannelUpdateData body = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/default-channels/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Patch; + + if (body != default(Models.DefaultChannelUpdateData)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnUpdateFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnUpdateFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnUpdateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedUpdateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetRequest(RestApiException ex); + + public async Task GetAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/default-channels/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedDeleteRequest(RestApiException ex); + + public async Task DeleteAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/default-channels/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnDeleteFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnDeleteFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedDeleteRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Goal.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Goal.cs new file mode 100644 index 0000000000..979aa69c86 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Goal.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IGoal + { + Task CreateAsync( + Models.GoalRequestJson body, + int definitionId, + string channelName, + CancellationToken cancellationToken = default + ); + + Task GetGoalTimesAsync( + int definitionId, + string channelName, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Goal : IServiceOperations, IGoal + { + public Goal(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedCreateRequest(RestApiException ex); + + public async Task CreateAsync( + Models.GoalRequestJson body, + int definitionId, + string channelName, + CancellationToken cancellationToken = default + ) + { + + if (body == default(Models.GoalRequestJson)) + { + throw new ArgumentNullException(nameof(body)); + } + + if (string.IsNullOrEmpty(channelName)) + { + throw new ArgumentNullException(nameof(channelName)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/goals/channelName/{channelName}/definitionId/{definitionId}".Replace("{channelName}", Uri.EscapeDataString(Client.Serialize(channelName))).Replace("{definitionId}", Uri.EscapeDataString(Client.Serialize(definitionId))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Put; + + if (body != default(Models.GoalRequestJson)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnCreateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetGoalTimesRequest(RestApiException ex); + + public async Task GetGoalTimesAsync( + int definitionId, + string channelName, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(channelName)) + { + throw new ArgumentNullException(nameof(channelName)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/goals/channelName/{channelName}/definitionId/{definitionId}".Replace("{definitionId}", Uri.EscapeDataString(Client.Serialize(definitionId))).Replace("{channelName}", Uri.EscapeDataString(Client.Serialize(channelName))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetGoalTimesFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetGoalTimesFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetGoalTimesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetGoalTimesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ApiError.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ApiError.cs new file mode 100644 index 0000000000..9a256a509e --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ApiError.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class ApiError + { + public ApiError() + { + } + + [JsonProperty("message")] + public string Message { get; set; } + + [JsonProperty("errors")] + public IImmutableList Errors { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Asset.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Asset.cs new file mode 100644 index 0000000000..6eb5b33346 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Asset.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Asset + { + public Asset(int id, int buildId, bool nonShipping, string name, string version, IImmutableList locations) + { + Id = id; + BuildId = buildId; + NonShipping = nonShipping; + Name = name; + Version = version; + Locations = locations; + } + + [JsonProperty("id")] + public int Id { get; } + + [JsonProperty("name")] + public string Name { get; } + + [JsonProperty("version")] + public string Version { get; } + + [JsonProperty("buildId")] + public int BuildId { get; set; } + + [JsonProperty("nonShipping")] + public bool NonShipping { get; set; } + + [JsonProperty("locations")] + public IImmutableList Locations { get; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetAndLocation.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetAndLocation.cs new file mode 100644 index 0000000000..01bada8070 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetAndLocation.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class AssetAndLocation + { + public AssetAndLocation(int assetId, Models.LocationType locationType) + { + AssetId = assetId; + LocationType = locationType; + } + + [JsonProperty("assetId")] + public int AssetId { get; set; } + + [JsonProperty("location")] + public string Location { get; set; } + + [JsonProperty("locationType")] + public Models.LocationType LocationType { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (LocationType == default) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetData.cs new file mode 100644 index 0000000000..e13b61a5c1 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetData.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class AssetData + { + public AssetData(bool nonShipping) + { + NonShipping = nonShipping; + } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("version")] + public string Version { get; set; } + + [JsonProperty("nonShipping")] + public bool NonShipping { get; set; } + + [JsonProperty("locations")] + public IImmutableList Locations { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocation.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocation.cs new file mode 100644 index 0000000000..4a55f8bf39 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocation.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class AssetLocation + { + public AssetLocation(int id, Models.LocationType type, string location) + { + Id = id; + Type = type; + Location = location; + } + + [JsonProperty("id")] + public int Id { get; } + + [JsonProperty("location")] + public string Location { get; } + + [JsonProperty("type")] + public Models.LocationType Type { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (Type == default) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocationData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocationData.cs new file mode 100644 index 0000000000..82de28675c --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/AssetLocationData.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class AssetLocationData + { + public AssetLocationData(Models.LocationType type) + { + Type = type; + } + + [JsonProperty("location")] + public string Location { get; set; } + + [JsonProperty("type")] + public Models.LocationType Type { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (Type == default) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Build.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Build.cs new file mode 100644 index 0000000000..2fc6b6697a --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Build.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Build + { + public Build(int id, DateTimeOffset dateProduced, int staleness, bool released, bool stable, string commit, IImmutableList channels, IImmutableList assets, IImmutableList dependencies, IImmutableList incoherencies) + { + Id = id; + DateProduced = dateProduced; + Staleness = staleness; + Released = released; + Stable = stable; + Commit = commit; + Channels = channels; + Assets = assets; + Dependencies = dependencies; + Incoherencies = incoherencies; + } + + [JsonProperty("id")] + public int Id { get; } + + [JsonProperty("commit")] + public string Commit { get; } + + [JsonProperty("azureDevOpsBuildId")] + public int? AzureDevOpsBuildId { get; set; } + + [JsonProperty("azureDevOpsBuildDefinitionId")] + public int? AzureDevOpsBuildDefinitionId { get; set; } + + [JsonProperty("azureDevOpsAccount")] + public string AzureDevOpsAccount { get; set; } + + [JsonProperty("azureDevOpsProject")] + public string AzureDevOpsProject { get; set; } + + [JsonProperty("azureDevOpsBuildNumber")] + public string AzureDevOpsBuildNumber { get; set; } + + [JsonProperty("azureDevOpsRepository")] + public string AzureDevOpsRepository { get; set; } + + [JsonProperty("azureDevOpsBranch")] + public string AzureDevOpsBranch { get; set; } + + [JsonProperty("gitHubRepository")] + public string GitHubRepository { get; set; } + + [JsonProperty("gitHubBranch")] + public string GitHubBranch { get; set; } + + [JsonProperty("dateProduced")] + public DateTimeOffset DateProduced { get; } + + [JsonProperty("channels")] + public IImmutableList Channels { get; } + + [JsonProperty("assets")] + public IImmutableList Assets { get; } + + [JsonProperty("dependencies")] + public IImmutableList Dependencies { get; } + + [JsonProperty("incoherencies")] + public IImmutableList Incoherencies { get; } + + [JsonProperty("staleness")] + public int Staleness { get; } + + [JsonProperty("released")] + public bool Released { get; } + + [JsonProperty("stable")] + public bool Stable { get; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildData.cs new file mode 100644 index 0000000000..05cb82bebe --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildData.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildData + { + public BuildData(string commit, string azureDevOpsAccount, string azureDevOpsProject, string azureDevOpsBuildNumber, string azureDevOpsRepository, string azureDevOpsBranch, bool released, bool stable) + { + Commit = commit; + AzureDevOpsAccount = azureDevOpsAccount; + AzureDevOpsProject = azureDevOpsProject; + AzureDevOpsBuildNumber = azureDevOpsBuildNumber; + AzureDevOpsRepository = azureDevOpsRepository; + AzureDevOpsBranch = azureDevOpsBranch; + Released = released; + Stable = stable; + } + + [JsonProperty("commit")] + public string Commit { get; set; } + + [JsonProperty("assets")] + public IImmutableList Assets { get; set; } + + [JsonProperty("dependencies")] + public IImmutableList Dependencies { get; set; } + + [JsonProperty("azureDevOpsBuildId")] + public int? AzureDevOpsBuildId { get; set; } + + [JsonProperty("azureDevOpsBuildDefinitionId")] + public int? AzureDevOpsBuildDefinitionId { get; set; } + + [JsonProperty("azureDevOpsAccount")] + public string AzureDevOpsAccount { get; set; } + + [JsonProperty("azureDevOpsProject")] + public string AzureDevOpsProject { get; set; } + + [JsonProperty("azureDevOpsBuildNumber")] + public string AzureDevOpsBuildNumber { get; set; } + + [JsonProperty("azureDevOpsRepository")] + public string AzureDevOpsRepository { get; set; } + + [JsonProperty("azureDevOpsBranch")] + public string AzureDevOpsBranch { get; set; } + + [JsonProperty("gitHubRepository")] + public string GitHubRepository { get; set; } + + [JsonProperty("gitHubBranch")] + public string GitHubBranch { get; set; } + + [JsonProperty("released")] + public bool Released { get; set; } + + [JsonProperty("stable")] + public bool Stable { get; set; } + + [JsonProperty("incoherencies")] + public IImmutableList Incoherencies { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(Commit)) + { + return false; + } + if (string.IsNullOrEmpty(AzureDevOpsAccount)) + { + return false; + } + if (string.IsNullOrEmpty(AzureDevOpsProject)) + { + return false; + } + if (string.IsNullOrEmpty(AzureDevOpsBuildNumber)) + { + return false; + } + if (string.IsNullOrEmpty(AzureDevOpsRepository)) + { + return false; + } + if (string.IsNullOrEmpty(AzureDevOpsBranch)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildGraph.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildGraph.cs new file mode 100644 index 0000000000..8712dbad89 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildGraph.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildGraph + { + public BuildGraph(IImmutableDictionary builds) + { + Builds = builds; + } + + [JsonProperty("builds")] + public IImmutableDictionary Builds { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (Builds == default(IImmutableDictionary)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildIncoherence.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildIncoherence.cs new file mode 100644 index 0000000000..8c6ffd295f --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildIncoherence.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildIncoherence + { + public BuildIncoherence() + { + } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("version")] + public string Version { get; set; } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("commit")] + public string Commit { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildRef.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildRef.cs new file mode 100644 index 0000000000..6cfe51e033 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildRef.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildRef + { + public BuildRef(int buildId, bool isProduct, double timeToInclusionInMinutes) + { + BuildId = buildId; + IsProduct = isProduct; + TimeToInclusionInMinutes = timeToInclusionInMinutes; + } + + [JsonProperty("buildId")] + public int BuildId { get; set; } + + [JsonProperty("isProduct")] + public bool IsProduct { get; set; } + + [JsonProperty("timeToInclusionInMinutes")] + public double TimeToInclusionInMinutes { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildTime.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildTime.cs new file mode 100644 index 0000000000..d4e3392b78 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildTime.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildTime + { + public BuildTime() + { + } + + [JsonProperty("defaultChannelId")] + public int? DefaultChannelId { get; set; } + + [JsonProperty("officialBuildTime")] + public double? OfficialBuildTime { get; set; } + + [JsonProperty("prBuildTime")] + public double? PrBuildTime { get; set; } + + [JsonProperty("goalTimeInMinutes")] + public int? GoalTimeInMinutes { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildUpdate.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildUpdate.cs new file mode 100644 index 0000000000..c2a1c0de09 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/BuildUpdate.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class BuildUpdate + { + public BuildUpdate() + { + } + + [JsonProperty("released")] + public bool? Released { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Channel.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Channel.cs new file mode 100644 index 0000000000..6fa5433d62 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Channel.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Channel + { + public Channel(int id, string name, string classification) + { + Id = id; + Name = name; + Classification = classification; + } + + [JsonProperty("id")] + public int Id { get; } + + [JsonProperty("name")] + public string Name { get; } + + [JsonProperty("classification")] + public string Classification { get; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(Name)) + { + return false; + } + if (string.IsNullOrEmpty(Classification)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/CodeFlowRequest.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/CodeFlowRequest.cs index f94cab6f82..a9e9c47afb 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/CodeFlowRequest.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/CodeFlowRequest.cs @@ -8,15 +8,17 @@ namespace ProductConstructionService.Client.Models { public partial class CodeFlowRequest { - public CodeFlowRequest() + public CodeFlowRequest(Guid subscriptionId, int buildId) { + SubscriptionId = subscriptionId; + BuildId = buildId; } [JsonProperty("subscriptionId")] - public Guid? SubscriptionId { get; set; } + public Guid SubscriptionId { get; set; } [JsonProperty("buildId")] - public int? BuildId { get; set; } + public int BuildId { get; set; } [JsonProperty("prBranch")] public string PrBranch { get; set; } diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Commit.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Commit.cs new file mode 100644 index 0000000000..b3ee81690d --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Commit.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Commit + { + public Commit() + { + } + + [JsonProperty("author")] + public string Author { get; set; } + + [JsonProperty("sha")] + public string Sha { get; set; } + + [JsonProperty("message")] + public string Message { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannel.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannel.cs new file mode 100644 index 0000000000..6e76b117f8 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannel.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class DefaultChannel + { + public DefaultChannel(int id, string repository, bool enabled) + { + Id = id; + Repository = repository; + Enabled = enabled; + } + + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("branch")] + public string Branch { get; set; } + + [JsonProperty("channel")] + public Models.Channel Channel { get; set; } + + [JsonProperty("enabled")] + public bool Enabled { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(Repository)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelCreateData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelCreateData.cs new file mode 100644 index 0000000000..f0c5393703 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelCreateData.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class DefaultChannelCreateData + { + public DefaultChannelCreateData(string repository, string branch, int channelId) + { + Repository = repository; + Branch = branch; + ChannelId = channelId; + } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("branch")] + public string Branch { get; set; } + + [JsonProperty("channelId")] + public int ChannelId { get; set; } + + [JsonProperty("enabled")] + public bool? Enabled { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(Repository)) + { + return false; + } + if (string.IsNullOrEmpty(Branch)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelUpdateData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelUpdateData.cs new file mode 100644 index 0000000000..0bf9ddee74 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/DefaultChannelUpdateData.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class DefaultChannelUpdateData + { + public DefaultChannelUpdateData() + { + } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("branch")] + public string Branch { get; set; } + + [JsonProperty("channelId")] + public int? ChannelId { get; set; } + + [JsonProperty("enabled")] + public bool? Enabled { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowEdge.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowEdge.cs new file mode 100644 index 0000000000..beaac8a9eb --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowEdge.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class FlowEdge + { + public FlowEdge(Guid subscriptionId, bool onLongestBuildPath, bool isToolingOnly, bool backEdge, string toId, string fromId) + { + SubscriptionId = subscriptionId; + OnLongestBuildPath = onLongestBuildPath; + IsToolingOnly = isToolingOnly; + BackEdge = backEdge; + ToId = toId; + FromId = fromId; + } + + [JsonProperty("toId")] + public string ToId { get; } + + [JsonProperty("fromId")] + public string FromId { get; } + + [JsonProperty("subscriptionId")] + public Guid SubscriptionId { get; set; } + + [JsonProperty("channelName")] + public string ChannelName { get; set; } + + [JsonProperty("onLongestBuildPath")] + public bool OnLongestBuildPath { get; set; } + + [JsonProperty("isToolingOnly")] + public bool IsToolingOnly { get; set; } + + [JsonProperty("partOfCycle")] + public bool? PartOfCycle { get; set; } + + [JsonProperty("backEdge")] + public bool BackEdge { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowGraph.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowGraph.cs new file mode 100644 index 0000000000..ead3669fba --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowGraph.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class FlowGraph + { + public FlowGraph(IImmutableList flowRefs, IImmutableList flowEdges) + { + FlowRefs = flowRefs; + FlowEdges = flowEdges; + } + + [JsonProperty("flowRefs")] + public IImmutableList FlowRefs { get; set; } + + [JsonProperty("flowEdges")] + public IImmutableList FlowEdges { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (FlowRefs == default(IImmutableList)) + { + return false; + } + if (FlowEdges == default(IImmutableList)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowRef.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowRef.cs new file mode 100644 index 0000000000..af9e904b14 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/FlowRef.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class FlowRef + { + public FlowRef(double officialBuildTime, double prBuildTime, bool onLongestBuildPath, double bestCasePathTime, double worstCasePathTime, int goalTimeInMinutes) + { + OfficialBuildTime = officialBuildTime; + PrBuildTime = prBuildTime; + OnLongestBuildPath = onLongestBuildPath; + BestCasePathTime = bestCasePathTime; + WorstCasePathTime = worstCasePathTime; + GoalTimeInMinutes = goalTimeInMinutes; + } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("branch")] + public string Branch { get; set; } + + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("officialBuildTime")] + public double OfficialBuildTime { get; set; } + + [JsonProperty("prBuildTime")] + public double PrBuildTime { get; set; } + + [JsonProperty("onLongestBuildPath")] + public bool OnLongestBuildPath { get; set; } + + [JsonProperty("bestCasePathTime")] + public double BestCasePathTime { get; set; } + + [JsonProperty("worstCasePathTime")] + public double WorstCasePathTime { get; set; } + + [JsonProperty("goalTimeInMinutes")] + public int GoalTimeInMinutes { get; set; } + + [JsonProperty("inputChannels")] + public IImmutableList InputChannels { get; set; } + + [JsonProperty("outputChannels")] + public IImmutableList OutputChannels { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Goal.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Goal.cs new file mode 100644 index 0000000000..f65fbf98ee --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Goal.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Goal + { + public Goal(int definitionId, int minutes) + { + DefinitionId = definitionId; + Minutes = minutes; + } + + [JsonProperty("definitionId")] + public int DefinitionId { get; set; } + + [JsonProperty("channel")] + public Models.Channel Channel { get; set; } + + [JsonProperty("minutes")] + public int Minutes { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/GoalRequestJson.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/GoalRequestJson.cs new file mode 100644 index 0000000000..9528c6d0e1 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/GoalRequestJson.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class GoalRequestJson + { + public GoalRequestJson(int minutes) + { + Minutes = minutes; + } + + [JsonProperty("minutes")] + public int Minutes { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/LocationType.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/LocationType.cs new file mode 100644 index 0000000000..5e78800006 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/LocationType.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; + +namespace ProductConstructionService.Client.Models +{ + public enum LocationType + { + [EnumMember(Value = "none")] + None, + [EnumMember(Value = "nugetFeed")] + NugetFeed, + [EnumMember(Value = "container")] + Container, + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/MergePolicy.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/MergePolicy.cs new file mode 100644 index 0000000000..a0b969fa86 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/MergePolicy.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class MergePolicy + { + public MergePolicy() + { + } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("properties")] + public IImmutableDictionary Properties { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ReleasePipeline.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ReleasePipeline.cs new file mode 100644 index 0000000000..329589cf44 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/ReleasePipeline.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class ReleasePipeline + { + public ReleasePipeline(int id, int pipelineIdentifier) + { + Id = id; + PipelineIdentifier = pipelineIdentifier; + } + + [JsonProperty("id")] + public int Id { get; set; } + + [JsonProperty("pipelineIdentifier")] + public int PipelineIdentifier { get; set; } + + [JsonProperty("organization")] + public string Organization { get; set; } + + [JsonProperty("project")] + public string Project { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryBranch.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryBranch.cs new file mode 100644 index 0000000000..5722c153cb --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryBranch.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class RepositoryBranch + { + public RepositoryBranch() + { + } + + [JsonProperty("repository")] + public string Repository { get; set; } + + [JsonProperty("branch")] + public string Branch { get; set; } + + [JsonProperty("mergePolicies")] + public IImmutableList MergePolicies { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryHistoryItem.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryHistoryItem.cs new file mode 100644 index 0000000000..4ddabe36eb --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/RepositoryHistoryItem.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class RepositoryHistoryItem + { + public RepositoryHistoryItem(DateTimeOffset timestamp, bool success, string repositoryName, string branchName, string errorMessage, string action, string retryUrl) + { + Timestamp = timestamp; + Success = success; + RepositoryName = repositoryName; + BranchName = branchName; + ErrorMessage = errorMessage; + Action = action; + RetryUrl = retryUrl; + } + + [JsonProperty("repositoryName")] + public string RepositoryName { get; } + + [JsonProperty("branchName")] + public string BranchName { get; } + + [JsonProperty("timestamp")] + public DateTimeOffset Timestamp { get; } + + [JsonProperty("errorMessage")] + public string ErrorMessage { get; } + + [JsonProperty("success")] + public bool Success { get; } + + [JsonProperty("action")] + public string Action { get; } + + [JsonProperty("retryUrl")] + public string RetryUrl { get; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Subscription.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Subscription.cs new file mode 100644 index 0000000000..c2ae2dde88 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/Subscription.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class Subscription + { + public Subscription(Guid id, bool enabled, bool sourceEnabled, string sourceRepository, string targetRepository, string targetBranch, string sourceDirectory, string targetDirectory, string pullRequestFailureNotificationTags, IImmutableList excludedAssets) + { + Id = id; + Enabled = enabled; + SourceEnabled = sourceEnabled; + SourceRepository = sourceRepository; + TargetRepository = targetRepository; + TargetBranch = targetBranch; + SourceDirectory = sourceDirectory; + TargetDirectory = targetDirectory; + PullRequestFailureNotificationTags = pullRequestFailureNotificationTags; + ExcludedAssets = excludedAssets; + } + + [JsonProperty("id")] + public Guid Id { get; } + + [JsonProperty("channel")] + public Models.Channel Channel { get; set; } + + [JsonProperty("sourceRepository")] + public string SourceRepository { get; } + + [JsonProperty("targetRepository")] + public string TargetRepository { get; } + + [JsonProperty("targetBranch")] + public string TargetBranch { get; } + + [JsonProperty("policy")] + public Models.SubscriptionPolicy Policy { get; set; } + + [JsonProperty("lastAppliedBuild")] + public Models.Build LastAppliedBuild { get; set; } + + [JsonProperty("enabled")] + public bool Enabled { get; } + + [JsonProperty("sourceEnabled")] + public bool SourceEnabled { get; } + + [JsonProperty("sourceDirectory")] + public string SourceDirectory { get; } + + [JsonProperty("targetDirectory")] + public string TargetDirectory { get; } + + [JsonProperty("pullRequestFailureNotificationTags")] + public string PullRequestFailureNotificationTags { get; } + + [JsonProperty("excludedAssets")] + public IImmutableList ExcludedAssets { get; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionData.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionData.cs new file mode 100644 index 0000000000..b85b38ca2f --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionData.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class SubscriptionData + { + public SubscriptionData(string channelName, string sourceRepository, string targetRepository, string targetBranch, Models.SubscriptionPolicy policy) + { + ChannelName = channelName; + SourceRepository = sourceRepository; + TargetRepository = targetRepository; + TargetBranch = targetBranch; + Policy = policy; + } + + [JsonProperty("channelName")] + public string ChannelName { get; set; } + + [JsonProperty("sourceRepository")] + public string SourceRepository { get; set; } + + [JsonProperty("targetRepository")] + public string TargetRepository { get; set; } + + [JsonProperty("targetBranch")] + public string TargetBranch { get; set; } + + [JsonProperty("enabled")] + public bool? Enabled { get; set; } + + [JsonProperty("sourceEnabled")] + public bool? SourceEnabled { get; set; } + + [JsonProperty("sourceDirectory")] + public string SourceDirectory { get; set; } + + [JsonProperty("targetDirectory")] + public string TargetDirectory { get; set; } + + [JsonProperty("policy")] + public Models.SubscriptionPolicy Policy { get; set; } + + [JsonProperty("pullRequestFailureNotificationTags")] + public string PullRequestFailureNotificationTags { get; set; } + + [JsonProperty("excludedAssets")] + public IImmutableList ExcludedAssets { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(ChannelName)) + { + return false; + } + if (string.IsNullOrEmpty(SourceRepository)) + { + return false; + } + if (string.IsNullOrEmpty(TargetRepository)) + { + return false; + } + if (string.IsNullOrEmpty(TargetBranch)) + { + return false; + } + if (Policy == default(Models.SubscriptionPolicy)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionHistoryItem.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionHistoryItem.cs new file mode 100644 index 0000000000..f82ffc8bce --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionHistoryItem.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class SubscriptionHistoryItem + { + public SubscriptionHistoryItem(DateTimeOffset timestamp, bool success, Guid subscriptionId, string errorMessage, string action, string retryUrl) + { + Timestamp = timestamp; + Success = success; + SubscriptionId = subscriptionId; + ErrorMessage = errorMessage; + Action = action; + RetryUrl = retryUrl; + } + + [JsonProperty("timestamp")] + public DateTimeOffset Timestamp { get; } + + [JsonProperty("errorMessage")] + public string ErrorMessage { get; } + + [JsonProperty("success")] + public bool Success { get; } + + [JsonProperty("subscriptionId")] + public Guid SubscriptionId { get; } + + [JsonProperty("action")] + public string Action { get; } + + [JsonProperty("retryUrl")] + public string RetryUrl { get; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionPolicy.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionPolicy.cs new file mode 100644 index 0000000000..5961c46bb4 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionPolicy.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class SubscriptionPolicy + { + public SubscriptionPolicy(bool batchable, Models.UpdateFrequency updateFrequency) + { + Batchable = batchable; + UpdateFrequency = updateFrequency; + } + + [JsonProperty("batchable")] + public bool Batchable { get; set; } + + [JsonProperty("updateFrequency")] + public Models.UpdateFrequency UpdateFrequency { get; set; } + + [JsonProperty("mergePolicies")] + public IImmutableList MergePolicies { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (UpdateFrequency == default) + { + return false; + } + return true; + } + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionUpdate.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionUpdate.cs new file mode 100644 index 0000000000..04e7211a51 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/SubscriptionUpdate.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace ProductConstructionService.Client.Models +{ + public partial class SubscriptionUpdate + { + public SubscriptionUpdate() + { + } + + [JsonProperty("channelName")] + public string ChannelName { get; set; } + + [JsonProperty("sourceRepository")] + public string SourceRepository { get; set; } + + [JsonProperty("policy")] + public Models.SubscriptionPolicy Policy { get; set; } + + [JsonProperty("enabled")] + public bool? Enabled { get; set; } + + [JsonProperty("sourceEnabled")] + public bool? SourceEnabled { get; set; } + + [JsonProperty("pullRequestFailureNotificationTags")] + public string PullRequestFailureNotificationTags { get; set; } + + [JsonProperty("sourceDirectory")] + public string SourceDirectory { get; set; } + + [JsonProperty("targetDirectory")] + public string TargetDirectory { get; set; } + + [JsonProperty("excludedAssets")] + public IImmutableList ExcludedAssets { get; set; } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/UpdateFrequency.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/UpdateFrequency.cs new file mode 100644 index 0000000000..d74312f921 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Models/UpdateFrequency.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; + +namespace ProductConstructionService.Client.Models +{ + public enum UpdateFrequency + { + [EnumMember(Value = "none")] + None, + [EnumMember(Value = "everyDay")] + EveryDay, + [EnumMember(Value = "everyBuild")] + EveryBuild, + [EnumMember(Value = "twiceDaily")] + TwiceDaily, + [EnumMember(Value = "everyWeek")] + EveryWeek, + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Pipelines.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Pipelines.cs new file mode 100644 index 0000000000..3061564d97 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Pipelines.cs @@ -0,0 +1,373 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IPipelines + { + Task> ListAsync( + string organization = default, + int? pipelineIdentifier = default, + string project = default, + CancellationToken cancellationToken = default + ); + + Task CreatePipelineAsync( + string organization, + int pipelineIdentifier, + string project, + CancellationToken cancellationToken = default + ); + + Task GetPipelineAsync( + int id, + CancellationToken cancellationToken = default + ); + + Task DeletePipelineAsync( + int id, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Pipelines : IServiceOperations, IPipelines + { + public Pipelines(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListRequest(RestApiException ex); + + public async Task> ListAsync( + string organization = default, + int? pipelineIdentifier = default, + string project = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/pipelines", + false); + + if (pipelineIdentifier != default) + { + _url.AppendQuery("pipelineIdentifier", Client.Serialize(pipelineIdentifier)); + } + if (!string.IsNullOrEmpty(organization)) + { + _url.AppendQuery("organization", Client.Serialize(organization)); + } + if (!string.IsNullOrEmpty(project)) + { + _url.AppendQuery("project", Client.Serialize(project)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedCreatePipelineRequest(RestApiException ex); + + public async Task CreatePipelineAsync( + string organization, + int pipelineIdentifier, + string project, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(organization)) + { + throw new ArgumentNullException(nameof(organization)); + } + + if (string.IsNullOrEmpty(project)) + { + throw new ArgumentNullException(nameof(project)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/pipelines", + false); + + if (pipelineIdentifier != default) + { + _url.AppendQuery("pipelineIdentifier", Client.Serialize(pipelineIdentifier)); + } + if (!string.IsNullOrEmpty(organization)) + { + _url.AppendQuery("organization", Client.Serialize(organization)); + } + if (!string.IsNullOrEmpty(project)) + { + _url.AppendQuery("project", Client.Serialize(project)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreatePipelineFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnCreatePipelineFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnCreatePipelineFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreatePipelineRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetPipelineRequest(RestApiException ex); + + public async Task GetPipelineAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/pipelines/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetPipelineFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetPipelineFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetPipelineFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetPipelineRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedDeletePipelineRequest(RestApiException ex); + + public async Task DeletePipelineAsync( + int id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/pipelines/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnDeletePipelineFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnDeletePipelineFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnDeletePipelineFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedDeletePipelineRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/ProductConstructionServiceApi.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/ProductConstructionServiceApi.cs index 3f05855ea2..73f5d9ded6 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/Generated/ProductConstructionServiceApi.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/ProductConstructionServiceApi.cs @@ -26,8 +26,16 @@ public partial interface IProductConstructionServiceApi { ProductConstructionServiceApiOptions Options { get; set; } + IAssets Assets { get; } + IBuilds Builds { get; } + IBuildTime BuildTime { get; } ICodeFlow CodeFlow { get; } - IStatus Status { get; } + IDefaultChannels DefaultChannels { get; } + IGoal Goal { get; } + IChannels Channels { get; } + IPipelines Pipelines { get; } + IRepository Repository { get; } + ISubscriptions Subscriptions { get; } } public partial interface IServiceOperations @@ -38,7 +46,7 @@ public partial interface IServiceOperations public partial class ProductConstructionServiceApiOptions : ClientOptions { public ProductConstructionServiceApiOptions() - : this(new Uri("https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/")) + : this(new Uri("")) { } @@ -48,7 +56,7 @@ public ProductConstructionServiceApiOptions(Uri baseUri) } public ProductConstructionServiceApiOptions(TokenCredential credentials) - : this(new Uri("https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"), credentials) + : this(new Uri(""), credentials) { } @@ -103,9 +111,25 @@ public HttpPipeline Pipeline public JsonSerializerSettings SerializerSettings { get; } + public IAssets Assets { get; } + + public IBuilds Builds { get; } + + public IBuildTime BuildTime { get; } + public ICodeFlow CodeFlow { get; } - public IStatus Status { get; } + public IDefaultChannels DefaultChannels { get; } + + public IGoal Goal { get; } + + public IChannels Channels { get; } + + public IPipelines Pipelines { get; } + + public IRepository Repository { get; } + + public ISubscriptions Subscriptions { get; } public ProductConstructionServiceApi() @@ -116,8 +140,16 @@ public ProductConstructionServiceApi() public ProductConstructionServiceApi(ProductConstructionServiceApiOptions options) { Options = options; + Assets = new Assets(this); + Builds = new Builds(this); + BuildTime = new BuildTime(this); CodeFlow = new CodeFlow(this); - Status = new Status(this); + DefaultChannels = new DefaultChannels(this); + Goal = new Goal(this); + Channels = new Channels(this); + Pipelines = new Pipelines(this); + Repository = new Repository(this); + Subscriptions = new Subscriptions(this); SerializerSettings = new JsonSerializerSettings { Converters = diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Repository.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Repository.cs new file mode 100644 index 0000000000..efd210ddf3 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Repository.cs @@ -0,0 +1,465 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface IRepository + { + Task> ListRepositoriesAsync( + string branch = default, + string repository = default, + CancellationToken cancellationToken = default + ); + + Task> GetMergePoliciesAsync( + string branch, + string repository, + CancellationToken cancellationToken = default + ); + + Task SetMergePoliciesAsync( + string branch, + string repository, + IImmutableList body = default, + CancellationToken cancellationToken = default + ); + + AsyncPageable GetHistoryAsync( + string branch, + string repository, + CancellationToken cancellationToken = default + ); + + Task> GetHistoryPageAsync( + string branch, + string repository, + int? page = default, + int? perPage = default, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Repository : IServiceOperations, IRepository + { + public Repository(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListRepositoriesRequest(RestApiException ex); + + public async Task> ListRepositoriesAsync( + string branch = default, + string repository = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/repo-config/repositories", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("branch", Client.Serialize(branch)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListRepositoriesFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListRepositoriesFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListRepositoriesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListRepositoriesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetMergePoliciesRequest(RestApiException ex); + + public async Task> GetMergePoliciesAsync( + string branch, + string repository, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(branch)) + { + throw new ArgumentNullException(nameof(branch)); + } + + if (string.IsNullOrEmpty(repository)) + { + throw new ArgumentNullException(nameof(repository)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/repo-config/merge-policy", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("branch", Client.Serialize(branch)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetMergePoliciesFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetMergePoliciesFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnGetMergePoliciesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetMergePoliciesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedSetMergePoliciesRequest(RestApiException ex); + + public async Task SetMergePoliciesAsync( + string branch, + string repository, + IImmutableList body = default, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(branch)) + { + throw new ArgumentNullException(nameof(branch)); + } + + if (string.IsNullOrEmpty(repository)) + { + throw new ArgumentNullException(nameof(repository)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/repo-config/merge-policy", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("branch", Client.Serialize(branch)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + if (body != default(IImmutableList)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnSetMergePoliciesFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnSetMergePoliciesFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedSetMergePoliciesRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetHistoryRequest(RestApiException ex); + + public AsyncPageable GetHistoryAsync( + string branch, + string repository, + CancellationToken cancellationToken = default + ) + { + async IAsyncEnumerable> GetPages(string _continueToken, int? _pageSizeHint) + { + int? page = 1; + int? perPage = _pageSizeHint; + + if (!string.IsNullOrEmpty(_continueToken)) + { + page = int.Parse(_continueToken); + } + + while (true) + { + Page _page = null; + + try { + _page = await GetHistoryPageAsync( + branch, + repository, + page, + perPage, + cancellationToken + ).ConfigureAwait(false); + if (_page.Values.Count < 1) + { + yield break; + } + } + catch (RestApiException e) when (e.Response.Status == 404) + { + yield break; + } + + yield return _page; + page++; + } + } + return AsyncPageable.Create(GetPages); + } + + public async Task> GetHistoryPageAsync( + string branch, + string repository, + int? page = default, + int? perPage = default, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(branch)) + { + throw new ArgumentNullException(nameof(branch)); + } + + if (string.IsNullOrEmpty(repository)) + { + throw new ArgumentNullException(nameof(repository)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/repo-config/history", + false); + + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("branch", Client.Serialize(branch)); + } + if (page != default) + { + _url.AppendQuery("page", Client.Serialize(page)); + } + if (perPage != default) + { + _url.AppendQuery("perPage", Client.Serialize(perPage)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetHistoryFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetHistoryFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return Page.FromValues(_body, (page + 1).ToString(), _res); + } + } + } + } + + internal async Task OnGetHistoryFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetHistoryRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Status.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Status.cs deleted file mode 100644 index eab7b05c8c..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Status.cs +++ /dev/null @@ -1,208 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Azure; -using Azure.Core; - - - -namespace ProductConstructionService.Client -{ - public partial interface IStatus - { - Task StopAsync( - CancellationToken cancellationToken = default - ); - - Task StartAsync( - CancellationToken cancellationToken = default - ); - - Task Async( - CancellationToken cancellationToken = default - ); - - } - - internal partial class Status : IServiceOperations, IStatus - { - public Status(ProductConstructionServiceApi client) - { - Client = client ?? throw new ArgumentNullException(nameof(client)); - } - - public ProductConstructionServiceApi Client { get; } - - partial void HandleFailedRequest(RestApiException ex); - - partial void HandleFailedStopRequest(RestApiException ex); - - public async Task StopAsync( - CancellationToken cancellationToken = default - ) - { - - - var _baseUri = Client.Options.BaseUri; - var _url = new RequestUriBuilder(); - _url.Reset(_baseUri); - _url.AppendPath( - "/status/stop", - false); - - - - using (var _req = Client.Pipeline.CreateRequest()) - { - _req.Uri = _url; - _req.Method = RequestMethod.Put; - - using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) - { - if (_res.Status < 200 || _res.Status >= 300) - { - await OnStopFailed(_req, _res).ConfigureAwait(false); - } - - - return; - } - } - } - - internal async Task OnStopFailed(Request req, Response res) - { - string content = null; - if (res.ContentStream != null) - { - using (var reader = new StreamReader(res.ContentStream)) - { - content = await reader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var ex = new RestApiException( - req, - res, - content); - HandleFailedStopRequest(ex); - HandleFailedRequest(ex); - Client.OnFailedRequest(ex); - throw ex; - } - - partial void HandleFailedStartRequest(RestApiException ex); - - public async Task StartAsync( - CancellationToken cancellationToken = default - ) - { - - - var _baseUri = Client.Options.BaseUri; - var _url = new RequestUriBuilder(); - _url.Reset(_baseUri); - _url.AppendPath( - "/status/start", - false); - - - - using (var _req = Client.Pipeline.CreateRequest()) - { - _req.Uri = _url; - _req.Method = RequestMethod.Put; - - using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) - { - if (_res.Status < 200 || _res.Status >= 300) - { - await OnStartFailed(_req, _res).ConfigureAwait(false); - } - - - return; - } - } - } - - internal async Task OnStartFailed(Request req, Response res) - { - string content = null; - if (res.ContentStream != null) - { - using (var reader = new StreamReader(res.ContentStream)) - { - content = await reader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var ex = new RestApiException( - req, - res, - content); - HandleFailedStartRequest(ex); - HandleFailedRequest(ex); - Client.OnFailedRequest(ex); - throw ex; - } - - public async Task Async( - CancellationToken cancellationToken = default - ) - { - - - var _baseUri = Client.Options.BaseUri; - var _url = new RequestUriBuilder(); - _url.Reset(_baseUri); - _url.AppendPath( - "/status", - false); - - - - using (var _req = Client.Pipeline.CreateRequest()) - { - _req.Uri = _url; - _req.Method = RequestMethod.Get; - - using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) - { - if (_res.Status < 200 || _res.Status >= 300) - { - await OnFailed(_req, _res).ConfigureAwait(false); - } - - - return; - } - } - } - - internal async Task OnFailed(Request req, Response res) - { - string content = null; - if (res.ContentStream != null) - { - using (var reader = new StreamReader(res.ContentStream)) - { - content = await reader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var ex = new RestApiException( - req, - res, - content); - HandleFailedRequest(ex); - HandleFailedRequest(ex); - Client.OnFailedRequest(ex); - throw ex; - } - } -} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/Generated/Subscriptions.cs b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Subscriptions.cs new file mode 100644 index 0000000000..e7d4992476 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Client/Generated/Subscriptions.cs @@ -0,0 +1,752 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Core; + + + +namespace ProductConstructionService.Client +{ + public partial interface ISubscriptions + { + Task> ListSubscriptionsAsync( + bool? enabled = default, + int? channelId = default, + string sourceDirectory = default, + bool? sourceEnabled = default, + string sourceRepository = default, + string targetDirectory = default, + string targetRepository = default, + CancellationToken cancellationToken = default + ); + + Task CreateAsync( + Models.SubscriptionData body, + CancellationToken cancellationToken = default + ); + + Task GetSubscriptionAsync( + Guid id, + CancellationToken cancellationToken = default + ); + + Task UpdateSubscriptionAsync( + Guid id, + Models.SubscriptionUpdate body = default, + CancellationToken cancellationToken = default + ); + + Task DeleteSubscriptionAsync( + Guid id, + CancellationToken cancellationToken = default + ); + + Task TriggerSubscriptionAsync( + int barBuildId, + Guid id, + CancellationToken cancellationToken = default + ); + + Task TriggerDailyUpdateAsync( + CancellationToken cancellationToken = default + ); + + AsyncPageable GetSubscriptionHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ); + + Task> GetSubscriptionHistoryPageAsync( + Guid id, + int? page = default, + int? perPage = default, + CancellationToken cancellationToken = default + ); + + } + + internal partial class Subscriptions : IServiceOperations, ISubscriptions + { + public Subscriptions(ProductConstructionServiceApi client) + { + Client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public ProductConstructionServiceApi Client { get; } + + partial void HandleFailedRequest(RestApiException ex); + + partial void HandleFailedListSubscriptionsRequest(RestApiException ex); + + public async Task> ListSubscriptionsAsync( + bool? enabled = default, + int? channelId = default, + string sourceDirectory = default, + bool? sourceEnabled = default, + string sourceRepository = default, + string targetDirectory = default, + string targetRepository = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions", + false); + + if (!string.IsNullOrEmpty(sourceRepository)) + { + _url.AppendQuery("sourceRepository", Client.Serialize(sourceRepository)); + } + if (!string.IsNullOrEmpty(targetRepository)) + { + _url.AppendQuery("targetRepository", Client.Serialize(targetRepository)); + } + if (channelId != default) + { + _url.AppendQuery("channelId", Client.Serialize(channelId)); + } + if (enabled != default) + { + _url.AppendQuery("enabled", Client.Serialize(enabled)); + } + if (sourceEnabled != default) + { + _url.AppendQuery("sourceEnabled", Client.Serialize(sourceEnabled)); + } + if (!string.IsNullOrEmpty(sourceDirectory)) + { + _url.AppendQuery("sourceDirectory", Client.Serialize(sourceDirectory)); + } + if (!string.IsNullOrEmpty(targetDirectory)) + { + _url.AppendQuery("targetDirectory", Client.Serialize(targetDirectory)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnListSubscriptionsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnListSubscriptionsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnListSubscriptionsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedListSubscriptionsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedCreateRequest(RestApiException ex); + + public async Task CreateAsync( + Models.SubscriptionData body, + CancellationToken cancellationToken = default + ) + { + + if (body == default(Models.SubscriptionData)) + { + throw new ArgumentNullException(nameof(body)); + } + + if (!body.IsValid) + { + throw new ArgumentException("The parameter is not valid", nameof(body)); + } + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + if (body != default(Models.SubscriptionData)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnCreateFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnCreateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedCreateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetSubscriptionRequest(RestApiException ex); + + public async Task GetSubscriptionAsync( + Guid id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetSubscriptionFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetSubscriptionRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedUpdateSubscriptionRequest(RestApiException ex); + + public async Task UpdateSubscriptionAsync( + Guid id, + Models.SubscriptionUpdate body = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Patch; + + if (body != default(Models.SubscriptionUpdate)) + { + _req.Content = RequestContent.Create(Encoding.UTF8.GetBytes(Client.Serialize(body))); + _req.Headers.Add("Content-Type", "application/json; charset=utf-8"); + } + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnUpdateSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnUpdateSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnUpdateSubscriptionFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedUpdateSubscriptionRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedDeleteSubscriptionRequest(RestApiException ex); + + public async Task DeleteSubscriptionAsync( + Guid id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnDeleteSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnDeleteSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnDeleteSubscriptionFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedDeleteSubscriptionRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedTriggerSubscriptionRequest(RestApiException ex); + + public async Task TriggerSubscriptionAsync( + int barBuildId, + Guid id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}/trigger".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + if (barBuildId != default) + { + _url.AppendQuery("bar-build-id", Client.Serialize(barBuildId)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnTriggerSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnTriggerSubscriptionFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnTriggerSubscriptionFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedTriggerSubscriptionRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedTriggerDailyUpdateRequest(RestApiException ex); + + public async Task TriggerDailyUpdateAsync( + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/triggerDaily", + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Post; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnTriggerDailyUpdateFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnTriggerDailyUpdateFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedTriggerDailyUpdateRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + + partial void HandleFailedGetSubscriptionHistoryRequest(RestApiException ex); + + public AsyncPageable GetSubscriptionHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ) + { + async IAsyncEnumerable> GetPages(string _continueToken, int? _pageSizeHint) + { + int? page = 1; + int? perPage = _pageSizeHint; + + if (!string.IsNullOrEmpty(_continueToken)) + { + page = int.Parse(_continueToken); + } + + while (true) + { + Page _page = null; + + try { + _page = await GetSubscriptionHistoryPageAsync( + id, + page, + perPage, + cancellationToken + ).ConfigureAwait(false); + if (_page.Values.Count < 1) + { + yield break; + } + } + catch (RestApiException e) when (e.Response.Status == 404) + { + yield break; + } + + yield return _page; + page++; + } + } + return AsyncPageable.Create(GetPages); + } + + public async Task> GetSubscriptionHistoryPageAsync( + Guid id, + int? page = default, + int? perPage = default, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}/history".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + if (page != default) + { + _url.AppendQuery("page", Client.Serialize(page)); + } + if (perPage != default) + { + _url.AppendQuery("perPage", Client.Serialize(perPage)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetSubscriptionHistoryFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetSubscriptionHistoryFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return Page.FromValues(_body, (page + 1).ToString(), _res); + } + } + } + } + + internal async Task OnGetSubscriptionHistoryFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetSubscriptionHistoryRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Client/PcsApiFactory.cs b/src/ProductConstructionService/ProductConstructionService.Client/PcsApiFactory.cs index 436e833a8c..acf6ec443e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/PcsApiFactory.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/PcsApiFactory.cs @@ -14,13 +14,16 @@ public static class PcsApiFactory /// /// Optional BAR token. When provided, will be used as the primary auth method. /// Managed Identity to use for the auth + /// Whether to include interactive login flows public static IProductConstructionServiceApi GetAuthenticated( string? accessToken, - string? managedIdentityId) + string? managedIdentityId, + bool disableInteractiveAuth) { return new ProductConstructionServiceApi(new ProductConstructionServiceApiOptions( accessToken, - managedIdentityId)); + managedIdentityId, + disableInteractiveAuth)); } /// @@ -44,14 +47,18 @@ public static IProductConstructionServiceApi GetAnonymous() /// You can get the access token by logging in to your ProductConstructionService instance /// and proceeding to Profile page. /// + /// Managed Identity to use for the auth + /// Whether to include interactive login flows public static IProductConstructionServiceApi GetAuthenticated( string baseUri, string? accessToken, - string? managedIdentityId) + string? managedIdentityId, + bool disableInteractiveAuth) { return new ProductConstructionServiceApi(new ProductConstructionServiceApiOptions( accessToken, - managedIdentityId)); + managedIdentityId, + disableInteractiveAuth)); } /// diff --git a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionService.Client.csproj b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionService.Client.csproj index 1316a3cd26..dcd797ad59 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionService.Client.csproj +++ b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionService.Client.csproj @@ -7,7 +7,7 @@ $(MSBuildProjectDirectory)\Generated ProductConstructionServiceApi - https://localhost:53180/swagger/v1/swagger.json + https://localhost:53180/swagger.json diff --git a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs index e27da4b9d6..11b43c1472 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs @@ -9,11 +9,31 @@ namespace ProductConstructionService.Client { public partial class ProductConstructionServiceApiOptions { - public const string StagingPcsBaseUri = "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; + // https://ms.portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/54c17f3d-7325-4eca-9db7-f090bfc765a8/isMSAApp~/false + private const string MaestroProductionAppId = "54c17f3d-7325-4eca-9db7-f090bfc765a8"; + + // https://ms.portal.azure.com/#view/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/~/Overview/appId/baf98f1b-374e-487d-af42-aa33807f11e4/isMSAApp~/false + private const string MaestroStagingAppId = "baf98f1b-374e-487d-af42-aa33807f11e4"; + + public const string ProductionMaestroUri = "https://maestro.dot.net/"; + public const string OldProductionMaestroUri = "https://maestro-prod.westus2.cloudapp.azure.com/"; + + public const string StagingMaestroUri = "https://maestro.int-dot.net/"; + public const string OldPcsStagingUri = "https://maestro-int.westus2.cloudapp.azure.com/"; + public const string PcsStagingUri = "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; + public const string PcsLocalUri = "https://localhost:53180/"; + + private const string APP_USER_SCOPE = "Maestro.User"; private static readonly Dictionary EntraAppIds = new Dictionary { - [StagingPcsBaseUri.TrimEnd('/')] = "baf98f1b-374e-487d-af42-aa33807f11e4", + [StagingMaestroUri.TrimEnd('/')] = MaestroStagingAppId, + [OldPcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, + [PcsStagingUri.TrimEnd('/')] = MaestroStagingAppId, + [PcsLocalUri.TrimEnd('/')] = MaestroStagingAppId, + + [ProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, + [OldProductionMaestroUri.TrimEnd('/')] = MaestroProductionAppId, }; /// @@ -22,15 +42,17 @@ public partial class ProductConstructionServiceApiOptions /// API base URI /// Optional BAR token. When provided, will be used as the primary auth method. /// Managed Identity to use for the auth - public ProductConstructionServiceApiOptions(string baseUri, string accessToken, string managedIdentityId) + /// Whether to include interactive login flows + public ProductConstructionServiceApiOptions(string baseUri, string accessToken, string managedIdentityId, bool disableInteractiveAuth) : this( new Uri(baseUri), AppCredentialResolver.CreateCredential( new AppCredentialResolverOptions(EntraAppIds[baseUri.TrimEnd('/')]) { - DisableInteractiveAuth = true, // the client is only used in Maestro for now + DisableInteractiveAuth = disableInteractiveAuth, Token = accessToken, ManagedIdentityId = managedIdentityId, + UserScope = APP_USER_SCOPE, })) { } @@ -38,11 +60,11 @@ public ProductConstructionServiceApiOptions(string baseUri, string accessToken, /// /// Creates a new instance of with the provided base URI. /// - /// API base URI /// Optional BAR token. When provided, will be used as the primary auth method. /// Managed Identity to use for the auth - public ProductConstructionServiceApiOptions(string accessToken, string managedIdentityId) - : this(StagingPcsBaseUri, accessToken, managedIdentityId) + /// Whether to include interactive login flows + public ProductConstructionServiceApiOptions(string accessToken, string managedIdentityId, bool disableInteractiveAuth) + : this(PcsStagingUri, accessToken, managedIdentityId, disableInteractiveAuth) { } } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs index 47b7647995..6ff40b864f 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs @@ -86,7 +86,7 @@ public void DeleteCurrentTestDirectory() buildAssetRegistryPat: null, managedIdentityId: null, disableInteractiveAuth: true, - buildAssetRegistryBaseUri: MaestroApiOptions.StagingBuildAssetRegistryBaseUri)); + buildAssetRegistryBaseUri: MaestroApiOptions.StagingMaestroUri)); protected static List GetExpectedFilesInVmr( NativePath vmrPath, diff --git a/test/SubscriptionActorService.Tests/PullRequestActorTests.cs b/test/SubscriptionActorService.Tests/PullRequestActorTests.cs index 0f1bad393c..3ecf6d93a1 100644 --- a/test/SubscriptionActorService.Tests/PullRequestActorTests.cs +++ b/test/SubscriptionActorService.Tests/PullRequestActorTests.cs @@ -23,11 +23,11 @@ using Moq; using NUnit.Framework; using ProductConstructionService.Client; -using ProductConstructionService.Client.Models; using SubscriptionActorService.StateModel; using Asset = Maestro.Contracts.Asset; using AssetData = Microsoft.DotNet.Maestro.Client.Models.AssetData; +using CodeFlowRequest = ProductConstructionService.Client.Models.CodeFlowRequest; using SynchronizePullRequestAction = System.Linq.Expressions.Expression>>>; namespace SubscriptionActorService.Tests; @@ -227,7 +227,7 @@ protected void ThenPcsShouldNotHaveBeenCalled(Build build, string? prUrl = null) var pcsRequests = new List(); _pcsClientCodeFlow .Verify( - r => r.FlowAsync( + r => r.FlowBuildAsync( It.Is(request => request.BuildId == build.Id && (prUrl == null || request.PrUrl == prUrl)), It.IsAny()), Times.Never); @@ -240,16 +240,14 @@ protected void AndPcsShouldHaveBeenCalled(Build build, string? prUrl, out string { var pcsRequests = new List(); _pcsClientCodeFlow - .Verify(r => r.FlowAsync(Capture.In(pcsRequests), It.IsAny())); + .Verify(r => r.FlowBuildAsync(Capture.In(pcsRequests), It.IsAny())); pcsRequests.Should() .BeEquivalentTo( new List { - new() + new(Subscription.Id, build.Id) { - SubscriptionId = Subscription.Id, - BuildId = build.Id, PrUrl = prUrl, } }, @@ -261,7 +259,7 @@ protected void AndPcsShouldHaveBeenCalled(Build build, string? prUrl, out string protected void ExpectPcsToGetCalled(Build build, string? prUrl = null) { _pcsClientCodeFlow - .Setup(r => r.FlowAsync( + .Setup(r => r.FlowBuildAsync( It.Is(r => r.BuildId == build.Id && r.SubscriptionId == Subscription.Id && r.PrUrl == prUrl), It.IsAny())) .Returns(Task.CompletedTask) From 51912f49d8a55c31615c992c06788e9c84ecff12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 29 Aug 2024 14:22:10 +0200 Subject: [PATCH 13/33] Copy `Maestro.ScenarioTests` to `ProductConstructionService.ScenarioTests` (#3890) --- arcade-services.sln | 15 + .../PcsStartup.cs | 1 + .../appsettings.Development.json | 2 +- .../Properties/serviceDependencies.json | 7 - .../Properties/serviceDependencies.local.json | 7 - .../AsyncDisposable.cs | 25 + .../AsyncDisposableValue.cs | 36 + .../Disposable.cs | 24 + .../EndToEndFlowLogic.cs | 495 +++++++++ .../ObjectHelpers/BuildRefComparer.cs | 27 + .../DependencyCollectionStringBuilder.cs | 23 + .../ObjectHelpers/SubscriptionBuilder.cs | 94 ++ ...ctConstructionService.ScenarioTests.csproj | 32 + .../ScenarioTestBase.cs | 946 ++++++++++++++++++ .../ScenarioTests_AzDoFlow.cs | 262 +++++ .../ScenarioTests_Builds.cs | 106 ++ .../ScenarioTests_Channels.cs | 46 + .../ScenarioTests_Clone.cs | 202 ++++ .../ScenarioTests_DefaultChannels.cs | 67 ++ .../ScenarioTests_Dependencies.cs | 96 ++ .../ScenarioTests_GitHubFlow.cs | 318 ++++++ .../ScenarioTests_MergePolicies.cs | 130 +++ .../ScenarioTests_RepoPolicies.cs | 61 ++ .../ScenarioTests_SdkUpdate.cs | 136 +++ .../ScenarioTests_Subscriptions.cs | 273 +++++ .../Shareable.cs | 41 + .../TemporaryDirectory.cs | 41 + .../TestHelpers.cs | 162 +++ .../TestHelpersTest.cs | 34 + .../TestParameters.cs | 179 ++++ .../TestRepository.cs | 18 + 31 files changed, 3891 insertions(+), 15 deletions(-) delete mode 100644 test/Maestro.ScenarioTests/Properties/serviceDependencies.json delete mode 100644 test/Maestro.ScenarioTests/Properties/serviceDependencies.local.json create mode 100644 test/ProductConstructionService.ScenarioTests/AsyncDisposable.cs create mode 100644 test/ProductConstructionService.ScenarioTests/AsyncDisposableValue.cs create mode 100644 test/ProductConstructionService.ScenarioTests/Disposable.cs create mode 100644 test/ProductConstructionService.ScenarioTests/EndToEndFlowLogic.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ObjectHelpers/BuildRefComparer.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ObjectHelpers/DependencyCollectionStringBuilder.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ObjectHelpers/SubscriptionBuilder.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_Builds.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_Clone.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_MergePolicies.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs create mode 100644 test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs create mode 100644 test/ProductConstructionService.ScenarioTests/Shareable.cs create mode 100644 test/ProductConstructionService.ScenarioTests/TemporaryDirectory.cs create mode 100644 test/ProductConstructionService.ScenarioTests/TestHelpers.cs create mode 100644 test/ProductConstructionService.ScenarioTests/TestHelpersTest.cs create mode 100644 test/ProductConstructionService.ScenarioTests/TestParameters.cs create mode 100644 test/ProductConstructionService.ScenarioTests/TestRepository.cs diff --git a/arcade-services.sln b/arcade-services.sln index bf6cf67856..39b3d91126 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -138,6 +138,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.DependencyFlow.Tests", "test\ProductConstructionService.DependencyFlow.Tests\ProductConstructionService.DependencyFlow.Tests.csproj", "{045BBB97-5A75-443A-8447-2035CBC0549A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.ScenarioTests", "test\ProductConstructionService.ScenarioTests\ProductConstructionService.ScenarioTests.csproj", "{12D91D30-EC50-4D2B-8D67-0C19FCD2303F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -665,6 +667,18 @@ Global {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x64.Build.0 = Release|Any CPU {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.ActiveCfg = Release|Any CPU {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.Build.0 = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|x64.ActiveCfg = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|x64.Build.0 = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|x86.ActiveCfg = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|x86.Build.0 = Debug|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|Any CPU.Build.0 = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|x64.ActiveCfg = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|x64.Build.0 = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|x86.ActiveCfg = Release|Any CPU + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -717,6 +731,7 @@ Global {29A75658-2DC4-4E85-8A53-97198F00F28D} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} {E312686C-A134-486F-9F62-89CE6CA34702} = {243A4561-BF35-405A-AF12-AC57BB27796D} {045BBB97-5A75-443A-8447-2035CBC0549A} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} + {12D91D30-EC50-4D2B-8D67-0C19FCD2303F} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {32B9C883-432E-4FC8-A1BF-090EB033DD5B} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index b1c1eacba7..3eec5f92df 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -103,6 +103,7 @@ static PcsStartup() (sub.SourceRepository == entity.Build.GitHubRepository || sub.SourceDirectory == entity.Build.AzureDevOpsRepository) && JsonExtensions.JsonValue(sub.PolicyString, "lax $.UpdateFrequency") == ((int)UpdateFrequency.EveryBuild).ToString()) // TODO (https://github.com/dotnet/arcade-services/issues/3880) + .ToList() .Where(sub => subscriptionIdGenerator.ShouldTriggerSubscription(sub.Id)) .ToList(); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json index 52397e3ac4..13496043ed 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json @@ -36,6 +36,6 @@ "Scope": [ "api://baf98f1b-374e-487d-af42-aa33807f11e4/Maestro.User" ] }, "ApiRedirect": { - "Uri": "https://maestro.dot.net/" + "Uri": "https://maestro.int-dot.net/" } } diff --git a/test/Maestro.ScenarioTests/Properties/serviceDependencies.json b/test/Maestro.ScenarioTests/Properties/serviceDependencies.json deleted file mode 100644 index a4e7aa3d33..0000000000 --- a/test/Maestro.ScenarioTests/Properties/serviceDependencies.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "dependencies": { - "secrets1": { - "type": "secrets" - } - } -} \ No newline at end of file diff --git a/test/Maestro.ScenarioTests/Properties/serviceDependencies.local.json b/test/Maestro.ScenarioTests/Properties/serviceDependencies.local.json deleted file mode 100644 index 09b109bc6f..0000000000 --- a/test/Maestro.ScenarioTests/Properties/serviceDependencies.local.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "dependencies": { - "secrets1": { - "type": "secrets.user" - } - } -} \ No newline at end of file diff --git a/test/ProductConstructionService.ScenarioTests/AsyncDisposable.cs b/test/ProductConstructionService.ScenarioTests/AsyncDisposable.cs new file mode 100644 index 0000000000..9781906447 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/AsyncDisposable.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +public class AsyncDisposable : IAsyncDisposable +{ + private Func _dispose; + + public static IAsyncDisposable Create(Func dispose) + { + return new AsyncDisposable(dispose); + } + + private AsyncDisposable(Func dispose) + { + _dispose = dispose; + } + + public ValueTask DisposeAsync() + { + Func dispose = Interlocked.Exchange(ref _dispose, null); + return dispose?.Invoke() ?? new ValueTask(Task.CompletedTask); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/AsyncDisposableValue.cs b/test/ProductConstructionService.ScenarioTests/AsyncDisposableValue.cs new file mode 100644 index 0000000000..9f9ea9aadb --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/AsyncDisposableValue.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +public class AsyncDisposableValue : IAsyncDisposable +{ + private Func _dispose; + + public T Value { get; } + + internal AsyncDisposableValue(T value, Func dispose) + { + Value = value; + _dispose = dispose; + } + + public ValueTask DisposeAsync() + { + Func dispose = Interlocked.Exchange(ref _dispose, null); + return dispose?.Invoke(Value) ?? new ValueTask(Task.CompletedTask); + } +} + +public static class AsyncDisposableValue +{ + public static AsyncDisposableValue Create(T value, Func dispose) + { + return new AsyncDisposableValue(value, dispose); + } + + public static AsyncDisposableValue Create(T value, Func dispose) + { + return new AsyncDisposableValue(value, _ => dispose()); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/Disposable.cs b/test/ProductConstructionService.ScenarioTests/Disposable.cs new file mode 100644 index 0000000000..d463e79494 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/Disposable.cs @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +public class Disposable : IDisposable +{ + private Action _dispose; + + public static IDisposable Create(Action dispose) + { + return new Disposable(dispose); + } + + private Disposable(Action dispose) + { + _dispose = dispose; + } + + public void Dispose() + { + Interlocked.Exchange(ref _dispose, null)?.Invoke(); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/EndToEndFlowLogic.cs b/test/ProductConstructionService.ScenarioTests/EndToEndFlowLogic.cs new file mode 100644 index 0000000000..d4b89e1e07 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/EndToEndFlowLogic.cs @@ -0,0 +1,495 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Microsoft.DotNet.DarcLib; +using NUnit.Framework; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +internal class EndToEndFlowLogic : ScenarioTestBase +{ + private const string SourceBuildNumber = "654321"; + private const string Source2BuildNumber = "987654"; + private readonly TestParameters _parameters; + + public EndToEndFlowLogic(TestParameters parameters) + { + _parameters = parameters; + SetTestParameters(_parameters); + } + + public async Task DarcBatchedFlowTestBase( + string targetBranch, + string channelName, + IImmutableList source1Assets, + IImmutableList source2Assets, + List expectedDependencies, + bool isAzDoTest) + { + var source1RepoName = TestRepository.TestRepo1Name; + var source2RepoName = TestRepository.TestRepo3Name; + var targetRepoName = TestRepository.TestRepo2Name; + + var testChannelName = channelName; + var source1RepoUri = isAzDoTest ? GetAzDoRepoUrl(source1RepoName) : GetGitHubRepoUrl(source1RepoName); + var source2RepoUri = isAzDoTest ? GetAzDoRepoUrl(source2RepoName) : GetGitHubRepoUrl(source2RepoName); + var targetRepoUri = isAzDoTest ? GetAzDoRepoUrl(targetRepoName) : GetGitHubRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + TestContext.WriteLine($"Adding a subscription from {source1RepoName} to {targetRepoName}"); + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionAsync(testChannelName, source1RepoName, targetRepoName, targetBranch, + UpdateFrequency.None.ToString(), "maestro-auth-test", additionalOptions: ["--batchable"], + sourceIsAzDo: isAzDoTest, targetIsAzDo: isAzDoTest); + + TestContext.WriteLine($"Adding a subscription from {source2RepoName} to {targetRepoName}"); + await using AsyncDisposableValue subscription2Id = await CreateSubscriptionAsync(testChannelName, source2RepoName, targetRepoName, targetBranch, + UpdateFrequency.None.ToString(), "maestro-auth-test", additionalOptions: ["--batchable"], + sourceIsAzDo: isAzDoTest, targetIsAzDo: isAzDoTest); + + TestContext.WriteLine("Set up build1 for intake into target repository"); + Build build1 = await CreateBuildAsync(source1RepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, source1Assets); + await AddBuildToChannelAsync(build1.Id, testChannelName); + + TestContext.WriteLine("Set up build2 for intake into target repository"); + Build build2 = await CreateBuildAsync(source2RepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, source2Assets); + await AddBuildToChannelAsync(build2.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + + TemporaryDirectory reposFolder = isAzDoTest ? await CloneAzDoRepositoryAsync(targetRepoName) : await CloneRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. source1Assets], targetRepoUri); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. source2Assets], targetRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + await TriggerSubscriptionAsync(subscription2Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + if (isAzDoTest) + { + await CheckBatchedAzDoPullRequest([source1RepoName, source2RepoName], targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory); + } + else + { + await CheckBatchedGitHubPullRequest(targetBranch, [source1RepoName, source2RepoName], targetRepoName, expectedDependencies, reposFolder.Directory); + } + } + } + } + } + + public async Task NonBatchedGitHubFlowTestBase(string targetBranch, string channelName, IImmutableList sourceAssets, + List expectedDependencies, bool allChecks = false) + { + var targetRepoName = TestRepository.TestRepo2Name; + var sourceRepoName = TestRepository.TestRepo1Name; + + var testChannelName = channelName; + var sourceRepoUri = GetGitHubRepoUrl(sourceRepoName); + var targetRepoUri = GetGitHubRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionForEndToEndTests( + testChannelName, sourceRepoName, targetRepoName, targetBranch, allChecks, false); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, sourceAssets); + await AddBuildToChannelAsync(build.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. sourceAssets], sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: true, isUpdated: false); + } + } + } + } + + public async Task NonBatchedGitHubFlowCoherencyTestBase(string targetBranch, string channelName, IImmutableList sourceAssets, + IImmutableList childSourceAssets, List expectedDependencies, string coherentParent, bool allChecks) + { + var targetRepoName = TestRepository.TestRepo3Name; + var sourceRepoName = TestRepository.TestRepo2Name; + var childRepoName = TestRepository.TestRepo1Name; + + var testChannelName = channelName; + var sourceRepoUri = GetGitHubRepoUrl(sourceRepoName); + var targetRepoUri = GetGitHubRepoUrl(targetRepoName); + var childSourceRepoUri = GetGitHubRepoUrl(childRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionForEndToEndTests( + testChannelName, sourceRepoName, targetRepoName, targetBranch, allChecks, false); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, sourceAssets); + await AddBuildToChannelAsync(build.Id, testChannelName); + + Build build2 = await CreateBuildAsync(childSourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo2Commit, + Source2BuildNumber, childSourceAssets); + await AddBuildToChannelAsync(build2.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. sourceAssets], sourceRepoUri); + + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. childSourceAssets], childSourceRepoUri, coherentParent); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: allChecks, isUpdated: false); + } + } + } + } + + public async Task NonBatchedGitHubFlowCoherencyOnlyTestBase(string targetBranch, string channelName, IImmutableList sourceAssets, + IImmutableList childSourceAssets, List expectedNonCoherencyDependencies, + List expectedCoherencyDependencies, string coherentParent) + { + var targetRepoName = TestRepository.TestRepo3Name; + var sourceRepoName = TestRepository.TestRepo2Name; + var childRepoName = TestRepository.TestRepo1Name; + + var testChannelName = channelName; + var sourceRepoUri = GetGitHubRepoUrl(sourceRepoName); + var targetRepoUri = GetGitHubRepoUrl(targetRepoName); + var childSourceRepoUri = GetGitHubRepoUrl(childRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionAsync( + testChannelName, + sourceRepoName, + targetRepoName, + targetBranch, + UpdateFrequency.None.ToString(), + "maestro-auth-test", + additionalOptions: ["--validate-coherency"], + trigger: true, + sourceIsAzDo: false, + targetIsAzDo: false); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build1 = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, + SourceBuildNumber, sourceAssets); + await AddBuildToChannelAsync(build1.Id, testChannelName); + + Build build2 = await CreateBuildAsync(childSourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo2Commit, + Source2BuildNumber, childSourceAssets); + await AddBuildToChannelAsync(build2.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. sourceAssets], sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies 1"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedNonCoherencyDependencies, reposFolder.Directory, isCompleted: true, isUpdated: false); + + await RunGitAsync("checkout", targetBranch); + await RunGitAsync("pull", "origin", targetBranch); + + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. childSourceAssets], childSourceRepoUri, coherentParent); + await GitCommitAsync("Add dependencies 2"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + var expectedPRTitle = $"[{targetBranch}] Update dependencies to ensure coherency"; + await CheckGitHubPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedCoherencyDependencies, reposFolder.Directory, isCompleted: false, isUpdated: false); + } + } + } + } + } + + public async Task NonBatchedUpdatingGitHubFlowTestBase(string targetBranch, string channelName, IImmutableList source1Assets, IImmutableList source1AssetsUpdated, + List expectedDependencies, List expectedUpdatedDependencies, bool allChecks = false) + { + var targetRepoName = TestRepository.TestRepo2Name; + var sourceRepoName = TestRepository.TestRepo1Name; + var sourceBranch = TestRepository.SourceBranch; + + var testChannelName = channelName; + var sourceRepoUri = GetGitHubRepoUrl(sourceRepoName); + var targetRepoUri = GetGitHubRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionForEndToEndTests( + testChannelName, sourceRepoName, targetRepoName, targetBranch, allChecks, false); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, source1Assets); + await AddBuildToChannelAsync(build.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. source1Assets], sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, allChecks); + + TestContext.WriteLine("Set up another build for intake into target repository"); + Build build2 = await CreateBuildAsync(sourceRepoUri, sourceBranch, TestRepository.CoherencyTestRepo2Commit, Source2BuildNumber, source1AssetsUpdated); + await AddBuildToChannelAsync(build2.Id, testChannelName); + + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting for PR to be updated in {targetRepoUri}"); + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedUpdatedDependencies, reposFolder.Directory, allChecks, true); + + // Then remove the second build from the channel, trigger the sub again, and it should revert back to the original dependency set + TestContext.Write("Remove the build from the channel and verify that the original dependencies are restored"); + await DeleteBuildFromChannelAsync(build2.Id.ToString(), testChannelName); + + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting for PR to be updated in {targetRepoUri}"); + + await CheckNonBatchedGitHubPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, allChecks, true); + } + } + } + } + + public async Task NonBatchedUpdatingAzDoFlowTestBase(string targetBranch, string channelName, IImmutableList sourceAssets, IImmutableList updatedSourceAssets, + List expectedDependencies, List expectedUpdatedDependencies) + { + var targetRepoName = TestRepository.TestRepo2Name; + var sourceRepoName = TestRepository.TestRepo1Name; + var sourceBranch = TestRepository.SourceBranch; + + var testChannelName = channelName; + var sourceRepoUri = GetAzDoRepoUrl(sourceRepoName); + var targetRepoUri = GetAzDoRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionForEndToEndTests( + testChannelName, sourceRepoName, targetRepoName, targetBranch, false, true); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, sourceAssets); + await AddBuildToChannelAsync(build.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneAzDoRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. sourceAssets], sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + await CheckNonBatchedAzDoPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: false, isUpdated: false); + + TestContext.WriteLine("Set up another build for intake into target repository"); + Build build2 = await CreateBuildAsync(sourceRepoUri, sourceBranch, TestRepository.CoherencyTestRepo2Commit, Source2BuildNumber, updatedSourceAssets); + + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting for PR to be updated in {targetRepoUri}"); + await CheckNonBatchedAzDoPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedUpdatedDependencies, reposFolder.Directory, isCompleted: false, isUpdated: true); + + // Then remove the second build from the channel, trigger the sub again, and it should revert back to the original dependency set + TestContext.Write("Remove the build from the channel and verify that the original dependencies are restored"); + await DeleteBuildFromChannelAsync(build2.Id.ToString(), testChannelName); + + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting for PR to be updated in {targetRepoUri}"); + await CheckNonBatchedAzDoPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: false, isUpdated: true); + } + } + } + } + + public async Task NonBatchedAzDoFlowTestBase(string targetBranch, string channelName, IImmutableList sourceAssets, + List expectedDependencies, bool allChecks = false, bool isFeedTest = false, string[] expectedFeeds = null, string[] notExpectedFeeds = null) + { + var targetRepoName = TestRepository.TestRepo2Name; + var sourceRepoName = TestRepository.TestRepo1Name; + + var testChannelName = channelName; + var sourceRepoUri = GetAzDoRepoUrl(sourceRepoName); + var targetRepoUri = GetAzDoRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue testChannel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionForEndToEndTests( + testChannelName, sourceRepoName, targetRepoName, targetBranch, allChecks, true); + + TestContext.WriteLine("Set up build for intake into target repository"); + Build build = await CreateBuildAsync(sourceRepoUri, TestRepository.SourceBranch, TestRepository.CoherencyTestRepo1Commit, SourceBuildNumber, sourceAssets); + await AddBuildToChannelAsync(build.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + TemporaryDirectory reposFolder = await CloneAzDoRepositoryAsync(targetRepoName); + + using (ChangeDirectory(reposFolder.Directory)) + { + await using (await CheckoutBranchAsync(targetBranch)) + { + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(reposFolder.Directory, [.. sourceAssets], sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await GitCommitAsync("Add dependencies"); + + await using (await PushGitBranchAsync("origin", targetBranch)) + { + TestContext.WriteLine("Trigger the dependency update"); + await TriggerSubscriptionAsync(subscription1Id.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in {targetRepoUri}"); + + if (allChecks) + { + await CheckNonBatchedAzDoPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: true, isUpdated: false); + return; + } + + if (isFeedTest) + { + await CheckNonBatchedAzDoPullRequest(sourceRepoName, targetRepoName, targetBranch, expectedDependencies, reposFolder.Directory, isCompleted: false, isUpdated: false, expectedFeeds: expectedFeeds, notExpectedFeeds: notExpectedFeeds); + return; + } + } + } + } + } + + private async Task> CreateSubscriptionForEndToEndTests(string testChannelName, string sourceRepoName, + string targetRepoName, string targetBranch, bool allChecks, bool isAzDoTest) + { + if (allChecks) + { + return await CreateSubscriptionAsync( + testChannelName, + sourceRepoName, + targetRepoName, + targetBranch, + UpdateFrequency.None.ToString(), + "maestro-auth-test", + additionalOptions: ["--all-checks-passed", "--validate-coherency", "--ignore-checks", "license/cla"], + trigger: true, + sourceIsAzDo: isAzDoTest, + targetIsAzDo: isAzDoTest); + } + else + { + return await CreateSubscriptionAsync( + testChannelName, + sourceRepoName, + targetRepoName, + targetBranch, + UpdateFrequency.None.ToString(), + "maestro-auth-test", + sourceIsAzDo: isAzDoTest, + targetIsAzDo: isAzDoTest); + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ObjectHelpers/BuildRefComparer.cs b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/BuildRefComparer.cs new file mode 100644 index 0000000000..e5079747fa --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/BuildRefComparer.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests.ObjectHelpers; + +public class BuildRefComparer : IEqualityComparer, IComparer +{ + public int Compare(object x, object y) + { + return ((BuildRef)x).BuildId.CompareTo(((BuildRef)y).BuildId); + } + + public bool Equals(BuildRef x, BuildRef y) + { + return x.BuildId == y.BuildId && + x.IsProduct == y.IsProduct && + x.TimeToInclusionInMinutes == y.TimeToInclusionInMinutes; + } + + public int GetHashCode(BuildRef obj) + { + return HashCode.Combine(obj.BuildId, obj.IsProduct, obj.TimeToInclusionInMinutes); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ObjectHelpers/DependencyCollectionStringBuilder.cs b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/DependencyCollectionStringBuilder.cs new file mode 100644 index 0000000000..7837dbc4cc --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/DependencyCollectionStringBuilder.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; +using Microsoft.DotNet.Darc; +using Microsoft.DotNet.DarcLib; + +namespace ProductConstructionService.ScenarioTests.ObjectHelpers; + +public class DependencyCollectionStringBuilder +{ + internal static string GetString(List expectedDependencies) + { + var stringBuilder = new StringBuilder(); + + foreach (DependencyDetail dependency in expectedDependencies) + { + stringBuilder.AppendLine(UxHelpers.DependencyToString(dependency)); + } + + return stringBuilder.ToString(); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ObjectHelpers/SubscriptionBuilder.cs b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/SubscriptionBuilder.cs new file mode 100644 index 0000000000..d4ca89695b --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ObjectHelpers/SubscriptionBuilder.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.Maestro.Client.Models; +using Newtonsoft.Json.Linq; + +namespace ProductConstructionService.ScenarioTests.ObjectHelpers; + +public class SubscriptionBuilder +{ + /// + /// Creates a subscription object based on a standard set of test inputs + /// + public static Subscription BuildSubscription( + string repo1Uri, + string repo2Uri, + string targetBranch, + string channelName, + string subscriptionId, + UpdateFrequency updateFrequency, + bool batchable, + List mergePolicyNames = null, + List ignoreChecks = null, + string failureNotificationTags = null) + { + var expectedSubscription = new Subscription( + Guid.Parse(subscriptionId), + true, + false, + repo1Uri, + repo2Uri, + targetBranch, + pullRequestFailureNotificationTags: failureNotificationTags, + sourceDirectory: null, + targetDirectory: null, + excludedAssets: ImmutableList.Empty) + { + Channel = new Channel(42, channelName, "test"), + Policy = new SubscriptionPolicy(batchable, updateFrequency) + }; + + List mergePolicies = []; + + if (mergePolicyNames == null) + { + expectedSubscription.Policy.MergePolicies = mergePolicies.ToImmutableList(); + return expectedSubscription; + } + + if (mergePolicyNames.Contains(MergePolicyConstants.StandardMergePolicyName)) + { + mergePolicies.Add(new MergePolicy + { + Name = MergePolicyConstants.StandardMergePolicyName + }); + } + + if (mergePolicyNames.Contains(MergePolicyConstants.AllCheckSuccessfulMergePolicyName) && ignoreChecks.Any()) + { + mergePolicies.Add( + new MergePolicy + { + Name = MergePolicyConstants.AllCheckSuccessfulMergePolicyName, + Properties = ImmutableDictionary.Create() + .Add(MergePolicyConstants.IgnoreChecksMergePolicyPropertyName, JToken.FromObject(ignoreChecks)) + }); + } + + if (mergePolicyNames.Contains(MergePolicyConstants.NoRequestedChangesMergePolicyName)) + { + mergePolicies.Add( + new MergePolicy + { + Name = MergePolicyConstants.NoRequestedChangesMergePolicyName, + Properties = ImmutableDictionary.Create() + }); + } + + if (mergePolicyNames.Contains(MergePolicyConstants.ValidateCoherencyMergePolicyName)) + { + mergePolicies.Add( + new MergePolicy + { + Name = MergePolicyConstants.ValidateCoherencyMergePolicyName, + Properties = ImmutableDictionary.Create() + }); + } + + expectedSubscription.Policy.MergePolicies = mergePolicies.ToImmutableList(); + return expectedSubscription; + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj b/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj new file mode 100644 index 0000000000..41833fa9c0 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj @@ -0,0 +1,32 @@ + + + + net8.0 + enable + disable + + false + true + a523e3e9-b284-4c40-962d-e06de454891e + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs new file mode 100644 index 0000000000..bfc62caee6 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs @@ -0,0 +1,946 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Text; +using System.Text.RegularExpressions; +using FluentAssertions; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using NuGet.Configuration; +using NUnit.Framework; +using ProductConstructionService.Client; +using ProductConstructionService.Client.Models; +using ProductConstructionService.ScenarioTests.ObjectHelpers; + +[assembly: Parallelizable(ParallelScope.Fixtures)] + +#nullable enable +namespace ProductConstructionService.ScenarioTests; + +internal abstract class ScenarioTestBase +{ + private TestParameters _parameters = null!; + private List _baseDarcRunArgs = []; + // We need this for tests where we have multiple updates + private readonly Dictionary _lastUpdatedPrTimes = []; + + protected IProductConstructionServiceApi PcsApi => _parameters.PcsApi; + + protected Octokit.GitHubClient GitHubApi => _parameters.GitHubApi; + + protected AzureDevOpsClient AzDoClient => _parameters.AzDoClient; + + public void SetTestParameters(TestParameters parameters) + { + _parameters = parameters; + _baseDarcRunArgs = [ + "--bar-uri", _parameters.MaestroBaseUri, + "--github-pat", _parameters.GitHubToken, + "--azdev-pat", _parameters.AzDoToken, + _parameters.IsCI ? "--ci" : "" + ]; + + if (!string.IsNullOrEmpty(_parameters.MaestroToken)) + { + _baseDarcRunArgs.AddRange(["--p", _parameters.MaestroToken]); + } + } + + protected async Task WaitForPullRequestAsync(string targetRepo, string targetBranch) + { + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + + var attempts = 10; + while (attempts-- > 0) + { + IReadOnlyList prs = await GitHubApi.PullRequest.GetAllForRepository(repo.Id, new Octokit.PullRequestRequest + { + Base = targetBranch, + }); + + if (prs.Count == 1) + { + // We use this method when we're creating the PR, and when we're fetching the updated PR + // We only want to set the Creation time when we're creating it + if (!_lastUpdatedPrTimes.ContainsKey(prs[0].Id)) + { + _lastUpdatedPrTimes[prs[0].Id] = prs[0].CreatedAt; + } + return prs[0]; + } + + if (prs.Count > 1) + { + throw new ScenarioTestException($"More than one pull request found in {targetRepo} targeting {targetBranch}"); + } + + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + throw new ScenarioTestException($"No pull request was created in {targetRepo} targeting {targetBranch}"); + } + + private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + { + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + + while (attempts-- > 0) + { + pr = await GitHubApi.PullRequest.Get(repo.Id, pr.Number); + + if (_lastUpdatedPrTimes[pr.Id] != pr.UpdatedAt) + { + _lastUpdatedPrTimes[pr.Id] = pr.UpdatedAt; + return pr; + } + + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); + } + + private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + { + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + + while (attempts-- > 0) + { + TestContext.WriteLine($"Starting check for merge, attempts remaining {attempts}"); + pr = await GitHubApi.PullRequest.Get(repo.Id, pr.Number); + + if (pr.Merged == true) + { + return true; + } + + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not merged within {attempts} minutes"); + } + + private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string targetBranch) + { + var searchBaseUrl = GetAzDoRepoUrl(targetRepoName); + IEnumerable prs = new List(); + + var attempts = 10; + while (attempts-- > 0) + { + try + { + prs = await SearchPullRequestsAsync(searchBaseUrl, targetBranch); + } + catch (HttpRequestException ex) + { + // Returning a 404 is expected before the PR has been created + var logger = new NUnitLogger(); + logger.LogInformation($"Searching for AzDo pull requests returned an error: {ex.Message}"); + } + + if (prs.Count() == 1) + { + return prs.FirstOrDefault(); + } + + if (prs.Count() > 1) + { + throw new ScenarioTestException($"More than one pull request found in {targetRepoName} targeting {targetBranch}"); + } + + await Task.Delay(60 * 1000); + } + + throw new ScenarioTestException($"No pull request was created in {searchBaseUrl} targeting {targetBranch}"); + } + + private async Task> SearchPullRequestsAsync(string repoUri, string targetPullRequestBranch) + { + (var accountName, var projectName, var repoName) = AzureDevOpsClient.ParseRepoUri(repoUri); + var query = new StringBuilder(); + + AzureDevOpsPrStatus prStatus = AzureDevOpsPrStatus.Active; + query.Append($"searchCriteria.status={prStatus.ToString().ToLower()}"); + query.Append($"&searchCriteria.targetRefName=refs/heads/{targetPullRequestBranch}"); + + JObject content = await _parameters.AzDoClient.ExecuteAzureDevOpsAPIRequestAsync( + HttpMethod.Get, + accountName, + projectName, + $"_apis/git/repositories/{repoName}/pullrequests?{query}", + new NUnitLogger() + ); + + IEnumerable prs = content.Value("value")!.Select(r => r.Value("pullRequestId")); + + return prs; + } + + private async Task> GetAzDoPullRequestAsync(int pullRequestId, string targetRepoName, string targetBranch, bool isUpdated, string? expectedPRTitle = null) + { + var repoUri = GetAzDoRepoUrl(targetRepoName); + (var accountName, var projectName, var repoName) = AzureDevOpsClient.ParseRepoUri(repoUri); + var apiBaseUrl = GetAzDoApiRepoUrl(targetRepoName); + + if (string.IsNullOrEmpty(expectedPRTitle)) + { + throw new Exception($"{nameof(expectedPRTitle)} must be defined for AzDo PRs that require an update"); + } + + for (var tries = 10; tries > 0; tries--) + { + PullRequest pr = await AzDoClient.GetPullRequestAsync($"{apiBaseUrl}/pullRequests/{pullRequestId}"); + var trimmedTitle = Regex.Replace(pr.Title, @"\s+", " "); + + if (!isUpdated || trimmedTitle == expectedPRTitle) + { + return AsyncDisposableValue.Create(pr, async () => + { + TestContext.WriteLine($"Cleaning up pull request {pr.Title}"); + + try + { + JObject content = await _parameters.AzDoClient.ExecuteAzureDevOpsAPIRequestAsync( + HttpMethod.Patch, + accountName, + projectName, + $"_apis/git/repositories/{targetRepoName}/pullrequests/{pullRequestId}", + new NUnitLogger(), + "{ \"status\" : \"abandoned\"}" + ); + } + catch + { + // If this throws it means that it was cleaned up by a different clean up method first + } + }); + } + + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + throw new ScenarioTestException($"The created pull request for {targetRepoName} targeting {targetBranch} was not updated with subsequent subscriptions after creation"); + } + + protected async Task CheckBatchedGitHubPullRequest(string targetBranch, string[] sourceRepoNames, + string targetRepoName, List expectedDependencies, string repoDirectory) + { + var repoNames = sourceRepoNames + .Select(name => $"{_parameters.GitHubTestOrg}/{name}") + .OrderBy(s => s); + + var expectedPRTitle = $"[{targetBranch}] Update dependencies from {string.Join(", ", repoNames)}"; + await CheckGitHubPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, false, true); + } + + protected async Task CheckNonBatchedGitHubPullRequest(string sourceRepoName, string targetRepoName, string targetBranch, + List expectedDependencies, string repoDirectory, bool isCompleted = false, bool isUpdated = false) + { + var expectedPRTitle = $"[{targetBranch}] Update dependencies from {_parameters.GitHubTestOrg}/{sourceRepoName}"; + await CheckGitHubPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, isCompleted, isUpdated); + } + + protected async Task CheckGitHubPullRequest(string expectedPRTitle, string targetRepoName, string targetBranch, + List expectedDependencies, string repoDirectory, bool isCompleted, bool isUpdated) + { + TestContext.WriteLine($"Checking opened PR in {targetBranch} {targetRepoName}"); + Octokit.PullRequest pullRequest = isUpdated + ? await WaitForUpdatedPullRequestAsync(targetRepoName, targetBranch) + : await WaitForPullRequestAsync(targetRepoName, targetBranch); + + pullRequest.Title.Should().Be(expectedPRTitle); + + using (ChangeDirectory(repoDirectory)) + { + await ValidatePullRequestDependencies(pullRequest.Head.Ref, expectedDependencies); + + if (isCompleted) + { + TestContext.WriteLine($"Checking for automatic merging of PR in {targetBranch} {targetRepoName}"); + + await WaitForMergedPullRequestAsync(targetRepoName, targetBranch); + } + } + } + + protected async Task CheckBatchedAzDoPullRequest( + string[] sourceRepoNames, + string targetRepoName, + string targetBranch, + List expectedDependencies, + string repoDirectory, + bool complete = false) + { + var repoNames = sourceRepoNames + .Select(n => $"{_parameters.AzureDevOpsAccount}/{_parameters.AzureDevOpsProject}/{n}") + .OrderBy(s => s); + + var expectedPRTitle = $"[{targetBranch}] Update dependencies from {string.Join(", ", repoNames)}"; + await CheckAzDoPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, complete, true, null, null); + } + + protected async Task CheckNonBatchedAzDoPullRequest( + string sourceRepoName, + string targetRepoName, + string targetBranch, + List expectedDependencies, + string repoDirectory, + bool isCompleted = false, + bool isUpdated = false, + string[]? expectedFeeds = null, + string[]? notExpectedFeeds = null) + { + var expectedPRTitle = $"[{targetBranch}] Update dependencies from {_parameters.AzureDevOpsAccount}/{_parameters.AzureDevOpsProject}/{sourceRepoName}"; + // TODO (https://github.com/dotnet/arcade-services/issues/3149): I noticed we are not passing isCompleted further down - when I put it there the tests started failing - but we should fix this + await CheckAzDoPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, false, isUpdated, expectedFeeds, notExpectedFeeds); + } + + protected async Task CheckAzDoPullRequest( + string expectedPRTitle, + string targetRepoName, + string targetBranch, + List expectedDependencies, + string repoDirectory, + bool isCompleted, + bool isUpdated, + string[]? expectedFeeds, + string[]? notExpectedFeeds) + { + var targetRepoUri = GetAzDoApiRepoUrl(targetRepoName); + TestContext.WriteLine($"Checking Opened PR in {targetBranch} {targetRepoUri} ..."); + var pullRequestId = await GetAzDoPullRequestIdAsync(targetRepoName, targetBranch); + await using AsyncDisposableValue pullRequest = await GetAzDoPullRequestAsync(pullRequestId, targetRepoName, targetBranch, isUpdated, expectedPRTitle); + + var trimmedTitle = Regex.Replace(pullRequest.Value.Title, @"\s+", " "); + trimmedTitle.Should().Be(expectedPRTitle); + + PrStatus expectedPRState = isCompleted ? PrStatus.Closed : PrStatus.Open; + var prStatus = await AzDoClient.GetPullRequestStatusAsync(GetAzDoApiRepoUrl(targetRepoName) + $"/pullRequests/{pullRequestId}"); + prStatus.Should().Be(expectedPRState); + + using (ChangeDirectory(repoDirectory)) + { + await ValidatePullRequestDependencies(pullRequest.Value.HeadBranch, expectedDependencies); + + if (expectedFeeds != null && notExpectedFeeds != null) + { + TestContext.WriteLine("Validating Nuget feeds in PR branch"); + + ISettings settings = Settings.LoadSpecificSettings(@"./", "nuget.config"); + var packageSourceProvider = new PackageSourceProvider(settings); + IEnumerable sources = packageSourceProvider.LoadPackageSources().Select(p => p.Source); + + sources.Should().Contain(expectedFeeds); + sources.Should().NotContain(notExpectedFeeds); + } + } + } + + private async Task ValidatePullRequestDependencies(string pullRequestBaseBranch, List expectedDependencies, int tries = 1) + { + var triesRemaining = tries; + while (triesRemaining-- > 0) + { + await CheckoutRemoteBranchAsync(pullRequestBaseBranch); + await RunGitAsync("pull"); + + var actualDependencies = await RunDarcAsync("get-dependencies"); + var expectedDependenciesString = DependencyCollectionStringBuilder.GetString(expectedDependencies); + + actualDependencies.Should().Be(expectedDependenciesString); + } + } + + protected async Task GitCommitAsync(string message) + { + await RunGitAsync("commit", "-am", message); + } + + protected async Task PushGitBranchAsync(string remote, string branch) + { + await RunGitAsync("push", "-u", remote, branch); + return AsyncDisposable.Create(async () => + { + TestContext.WriteLine($"Cleaning up Remote branch {branch}"); + + try + { + await RunGitAsync("push", remote, "--delete", branch); + } + catch + { + // If this throws it means that it was cleaned up by a different clean up method first + } + }); + } + + protected static string GetRepoUrl(string org, string repository) + { + return $"https://github.com/{org}/{repository}"; + } + + protected string GetGitHubRepoUrl(string repository) + { + return GetRepoUrl(_parameters.GitHubTestOrg, repository); + } + + protected string GetRepoFetchUrl(string repository) + { + return GetRepoFetchUrl(_parameters.GitHubTestOrg, repository); + } + + protected string GetRepoFetchUrl(string org, string repository) + { + return $"https://{_parameters.GitHubUser}:{_parameters.GitHubToken}@github.com/{org}/{repository}"; + } + + protected static string GetAzDoRepoUrl(string repoName, string azdoAccount = "dnceng", string azdoProject = "internal") + { + return $"https://dev.azure.com/{azdoAccount}/{azdoProject}/_git/{repoName}"; + } + + protected static string GetAzDoApiRepoUrl(string repoName, string azdoAccount = "dnceng", string azdoProject = "internal") + { + return $"https://dev.azure.com/{azdoAccount}/{azdoProject}/_apis/git/repositories/{repoName}"; + } + + protected Task RunDarcAsyncWithInput(string input, params string[] args) + { + return TestHelpers.RunExecutableAsyncWithInput(_parameters.DarcExePath, input, + [ + .. args, + .. _baseDarcRunArgs, + ]); + } + + protected Task RunDarcAsync(params string[] args) + { + return TestHelpers.RunExecutableAsync(_parameters.DarcExePath, + [ + .. args, + .. _baseDarcRunArgs, + ]); + } + + protected Task RunGitAsync(params string[] args) + { + return TestHelpers.RunExecutableAsync(_parameters.GitExePath, args); + } + + protected async Task> CreateTestChannelAsync(string testChannelName) + { + var message = ""; + + try + { + message = await RunDarcAsync("delete-channel", "--name", testChannelName); + } + catch (ScenarioTestException) + { + // If there are subscriptions associated the the channel then a previous test clean up failed + // Run a subscription clean up and try again + try + { + await DeleteSubscriptionsForChannel(testChannelName); + await RunDarcAsync("delete-channel", "--name", testChannelName); + } + catch (ScenarioTestException) + { + // Otherwise ignore failures from delete-channel, its just a pre-cleanup that isn't really part of the test + // And if the test previously succeeded then it'll fail because the channel doesn't exist + } + } + + await RunDarcAsync("add-channel", "--name", testChannelName, "--classification", "test"); + + return AsyncDisposableValue.Create(testChannelName, async () => + { + TestContext.WriteLine($"Cleaning up Test Channel {testChannelName}"); + try + { + var doubleDelete = await RunDarcAsync("delete-channel", "--name", testChannelName); + } + catch (ScenarioTestException) + { + // Ignore failures from delete-channel on cleanup, this delete is here to ensure that the channel is deleted + // even if the test does not do an explicit delete as part of the test. Other failures are typical that the channel has already been deleted. + } + }); + } + protected async Task AddDependenciesToLocalRepo(string repoPath, string name, string repoUri, bool isToolset = false) + { + using (ChangeDirectory(repoPath)) + { + await RunDarcAsync(["add-dependency", "--name", name, "--type", isToolset ? "toolset" : "product", "--repo", repoUri, "--version", "0.0.1"]); + } + } + protected async Task GetTestChannelsAsync() + { + return await RunDarcAsync("get-channels"); + } + + protected async Task DeleteTestChannelAsync(string testChannelName) + { + await RunDarcAsync("delete-channel", "--name", testChannelName); + } + + protected async Task AddDefaultTestChannelAsync(string testChannelName, string repoUri, string branchName) + { + return await RunDarcAsync("add-default-channel", "--channel", testChannelName, "--repo", repoUri, "--branch", branchName, "-q"); + } + + protected async Task GetDefaultTestChannelsAsync(string repoUri, string branch) + { + return await RunDarcAsync("get-default-channels", "--source-repo", repoUri, "--branch", branch); + } + + protected async Task DeleteDefaultTestChannelAsync(string testChannelName, string repoUri, string branch) + { + await RunDarcAsync("delete-default-channel", "--channel", testChannelName, "--repo", repoUri, "--branch", branch); + } + + protected async Task> CreateSubscriptionAsync( + string sourceChannelName, + string sourceRepo, + string targetRepo, + string targetBranch, + string updateFrequency, + string sourceOrg = "dotnet", + List? additionalOptions = null, + bool sourceIsAzDo = false, + bool targetIsAzDo = false, + bool trigger = false) + { + var sourceUrl = sourceIsAzDo ? GetAzDoRepoUrl(sourceRepo) : GetRepoUrl(sourceOrg, sourceRepo); + var targetUrl = targetIsAzDo ? GetAzDoRepoUrl(targetRepo) : GetGitHubRepoUrl(targetRepo); + + string[] command = + [ + "add-subscription", "-q", + "--channel", sourceChannelName, + "--source-repo", sourceUrl, + "--target-repo", targetUrl, + "--target-branch", targetBranch, + "--update-frequency", updateFrequency, + trigger? "--trigger" : "--no-trigger", + .. additionalOptions ?? [] + ]; + + var output = await RunDarcAsync(command); + + Match match = Regex.Match(output, "Successfully created new subscription with id '([a-f0-9-]+)'"); + if (!match.Success) + { + throw new ScenarioTestException("Unable to create subscription."); + } + + var subscriptionId = match.Groups[1].Value; + return AsyncDisposableValue.Create(subscriptionId, async () => + { + TestContext.WriteLine($"Cleaning up Test Subscription {subscriptionId}"); + try + { + await RunDarcAsync("delete-subscriptions", "--id", subscriptionId, "--quiet"); + } + catch (ScenarioTestException) + { + // If this throws an exception the most likely cause is that the subscription was deleted as part of the test case + } + }); + } + + protected async Task> CreateSubscriptionAsync(string yamlDefinition) + { + var output = await RunDarcAsyncWithInput(yamlDefinition, "add-subscription", "-q", "--read-stdin", "--no-trigger"); + + Match match = Regex.Match(output, "Successfully created new subscription with id '([a-f0-9-]+)'"); + if (match.Success) + { + var subscriptionId = match.Groups[1].Value; + return AsyncDisposableValue.Create(subscriptionId, async () => + { + TestContext.WriteLine($"Cleaning up Test Subscription {subscriptionId}"); + try + { + await RunDarcAsync("delete-subscriptions", "--id", subscriptionId, "--quiet"); + } + catch (ScenarioTestException) + { + // If this throws an exception the most likely cause is that the subscription was deleted as part of the test case + } + }); + } + + throw new ScenarioTestException("Unable to create subscription."); + } + + protected async Task GetSubscriptionInfo(string subscriptionId) + { + return await RunDarcAsync("get-subscriptions", "--ids", subscriptionId); + } + + protected async Task GetSubscriptions(string channelName) + { + return await RunDarcAsync("get-subscriptions", "--channel", channelName); + } + + protected async Task SetSubscriptionStatusByChannel(bool enableSub, string channelName) + { + await RunDarcAsync("subscription-status", enableSub ? "--enable" : "-d", "--channel", channelName, "--quiet"); + } + + protected async Task SetSubscriptionStatusById(bool enableSub, string subscriptionId) + { + await RunDarcAsync("subscription-status", "--id", subscriptionId, enableSub ? "--enable" : "-d", "--quiet"); + } + + protected async Task DeleteSubscriptionsForChannel(string channelName) + { + return await RunDarcAsync("delete-subscriptions", "--channel", channelName, "--quiet"); + } + + protected async Task DeleteSubscriptionById(string subscriptionId) + { + return await RunDarcAsync("delete-subscriptions", "--id", subscriptionId, "--quiet"); + } + + protected Task CreateBuildAsync(string repositoryUrl, string branch, string commit, string buildNumber, IImmutableList assets) + { + return CreateBuildAsync(repositoryUrl, branch, commit, buildNumber, assets, ImmutableList.Empty); + } + + protected async Task CreateBuildAsync(string repositoryUrl, string branch, string commit, string buildNumber, IImmutableList assets, IImmutableList dependencies) + { + Build build = await PcsApi.Builds.CreateAsync(new BuildData( + commit: commit, + azureDevOpsAccount: _parameters.AzureDevOpsAccount, + azureDevOpsProject: _parameters.AzureDevOpsProject, + azureDevOpsBuildNumber: buildNumber, + azureDevOpsRepository: repositoryUrl, + azureDevOpsBranch: branch, + released: false, + stable: false) + { + AzureDevOpsBuildId = _parameters.AzureDevOpsBuildId, + AzureDevOpsBuildDefinitionId = _parameters.AzureDevOpsBuildDefinitionId, + GitHubRepository = repositoryUrl, + GitHubBranch = branch, + Assets = assets, + Dependencies = dependencies, + }); + + return build; + } + + protected async Task GetDarcBuildAsync(int buildId) + { + var buildString = await RunDarcAsync("get-build", "--id", buildId.ToString()); + return buildString; + } + + protected async Task UpdateBuildAsync(int buildId, string updateParams) + { + var buildString = await RunDarcAsync("update-build", "--id", buildId.ToString(), updateParams); + return buildString; + } + + protected async Task AddDependenciesToLocalRepo(string repoPath, List dependencies, string repoUri, string coherentParent = "") + { + using (ChangeDirectory(repoPath)) + { + foreach (AssetData asset in dependencies) + { + List parameters = ["add-dependency", "--name", asset.Name, "--type", "product", "--repo", repoUri]; + + if (!string.IsNullOrEmpty(coherentParent)) + { + parameters.Add("--coherent-parent"); + parameters.Add(coherentParent); + } + var parameterArr = parameters.ToArray(); + + await RunDarcAsync(parameterArr); + } + } + } + + protected async Task GatherDrop(int buildId, string outputDir, bool includeReleased, string extraAssetsRegex) + { + string[] args = ["gather-drop", "--id", buildId.ToString(), "--dry-run", "--output-dir", outputDir]; + + if (includeReleased) + { + args = [.. args, "--include-released"]; + } + + if (!string.IsNullOrEmpty(extraAssetsRegex)) + { + args = [.. args, "--always-download-asset-filters", extraAssetsRegex]; + } + + return await RunDarcAsync(args); + } + + protected async Task TriggerSubscriptionAsync(string subscriptionId) + { + await PcsApi.Subscriptions.TriggerSubscriptionAsync(0, Guid.Parse(subscriptionId)); + } + + protected async Task AddBuildToChannelAsync(int buildId, string channelName) + { + await RunDarcAsync("add-build-to-channel", "--id", buildId.ToString(), "--channel", channelName, "--skip-assets-publishing"); + return AsyncDisposable.Create(async () => + { + TestContext.WriteLine($"Removing build {buildId} from channel {channelName}"); + await RunDarcAsync("delete-build-from-channel", "--id", buildId.ToString(), "--channel", channelName); + }); + } + + protected async Task DeleteBuildFromChannelAsync(string buildId, string channelName) + { + await RunDarcAsync("delete-build-from-channel", "--id", buildId, "--channel", channelName); + } + + protected static IDisposable ChangeDirectory(string directory) + { + var old = Directory.GetCurrentDirectory(); + TestContext.WriteLine($"Switching to directory {directory}"); + Directory.SetCurrentDirectory(directory); + return Disposable.Create(() => + { + TestContext.WriteLine($"Switching back to directory {old}"); + Directory.SetCurrentDirectory(old); + }); + } + + protected Task CloneRepositoryAsync(string repository) + { + return CloneRepositoryAsync(_parameters.GitHubTestOrg, repository); + } + + protected async Task CloneRepositoryAsync(string org, string repository) + { + using var shareable = Shareable.Create(TemporaryDirectory.Get()); + var directory = shareable.Peek()!.Directory; + + var fetchUrl = GetRepoFetchUrl(org, repository); + await RunGitAsync("clone", "--quiet", fetchUrl, directory); + + using (ChangeDirectory(directory)) + { + await RunGitAsync("config", "user.email", $"{_parameters.GitHubUser}@test.com"); + await RunGitAsync("config", "user.name", _parameters.GitHubUser); + await RunGitAsync("config", "gc.auto", "0"); + await RunGitAsync("config", "advice.detachedHead", "false"); + await RunGitAsync("config", "color.ui", "false"); + } + + return shareable.TryTake()!; + } + + protected string GetAzDoRepoAuthUrl(string repoName) + { + return $"https://{_parameters.GitHubUser}:{_parameters.AzDoToken}@dev.azure.com/{_parameters.AzureDevOpsAccount}/{_parameters.AzureDevOpsProject}/_git/{repoName}"; + } + + protected async Task CloneAzDoRepositoryAsync(string repoName) + { + using var shareable = Shareable.Create(TemporaryDirectory.Get()); + var directory = shareable.Peek()!.Directory; + + var authUrl = GetAzDoRepoAuthUrl(repoName); + await RunGitAsync("clone", "--quiet", authUrl, directory); + + using (ChangeDirectory(directory)) + { + // The GitHubUser and AzDoUser have the same user name so this uses the existing parameter + await RunGitAsync("config", "user.email", $"{_parameters.GitHubUser}@test.com"); + await RunGitAsync("config", "user.name", _parameters.GitHubUser); + } + + return shareable.TryTake()!; + } + + protected async Task CloneRepositoryWithDarc(string repoName, string version, string reposToIgnore, bool includeToolset, int depth) + { + var sourceRepoUri = GetRepoUrl("dotnet", repoName); + + using var shareable = Shareable.Create(TemporaryDirectory.Get()); + var directory = shareable.Peek().Directory; + + var reposFolder = Path.Join(directory, "cloned-repos"); + var gitDirFolder = Path.Join(directory, "git-dirs"); + + // Clone repo + await RunDarcAsync("clone", "--repo", sourceRepoUri, "--version", version, "--git-dir-folder", gitDirFolder, "--ignore-repos", reposToIgnore, "--repos-folder", reposFolder, "--depth", depth.ToString(), includeToolset ? "--include-toolset" : ""); + + return shareable.TryTake()!; + } + + protected async Task CheckoutRemoteRefAsync(string commit) + { + await RunGitAsync("fetch", "origin", commit); + await RunGitAsync("checkout", commit); + } + + protected async Task CheckoutRemoteBranchAsync(string branchName) + { + await RunGitAsync("fetch", "origin", branchName); + await RunGitAsync("checkout", branchName); + } + + protected async Task CheckoutBranchAsync(string branchName) + { + await RunGitAsync("fetch", "origin"); + await RunGitAsync("checkout", "-b", branchName); + + return AsyncDisposable.Create(async () => + { + TestContext.WriteLine($"Deleting remote branch {branchName}"); + try + { + await DeleteBranchAsync(branchName); + } + catch + { + // If this throws it means that it was cleaned up by a different clean up method first + } + }); + } + + protected async Task DeleteBranchAsync(string branchName) + { + await RunGitAsync("push", "origin", "--delete", branchName); + } + + protected static IImmutableList GetAssetData(string asset1Name, string asset1Version, string asset2Name, string asset2Version) + { + var location = @"https://pkgs.dev.azure.com/dnceng/public/_packaging/NotARealFeed/nuget/v3/index.json"; + LocationType locationType = LocationType.NugetFeed; + + AssetData asset1 = GetAssetDataWithLocations(asset1Name, asset1Version, location, locationType); + + AssetData asset2 = GetAssetDataWithLocations(asset2Name, asset2Version, location, locationType); + + return ImmutableList.Create(asset1, asset2); + } + + protected static AssetData GetAssetDataWithLocations( + string assetName, + string assetVersion, + string assetLocation1, + LocationType assetLocationType1, + string? assetLocation2 = null, + LocationType assetLocationType2 = LocationType.None) + { + var locationsListBuilder = ImmutableList.CreateBuilder(); + + var location1 = new AssetLocationData(assetLocationType1) + { Location = assetLocation1 }; + locationsListBuilder.Add(location1); + + if (assetLocation2 != null && assetLocationType2 != LocationType.None) + { + var location2 = new AssetLocationData(assetLocationType2) + { Location = assetLocation2 }; + locationsListBuilder.Add(location2); + } + + var asset = new AssetData(false) + { + Name = assetName, + Version = assetVersion, + Locations = locationsListBuilder.ToImmutable() + }; + + return asset; + } + + protected static IImmutableList GetSingleAssetData(string assetName, string assetVersion) + { + var asset = new AssetData(false) + { + Name = assetName, + Version = assetVersion, + Locations = ImmutableList.Create(new AssetLocationData(LocationType.NugetFeed) + { Location = @"https://pkgs.dev.azure.com/dnceng/public/_packaging/NotARealFeed/nuget/v3/index.json" }) + }; + + return ImmutableList.Create(asset); + } + + protected async Task SetRepositoryPolicies(string repoUri, string branchName, string[]? policyParams = null) + { + string[] commandParams = ["set-repository-policies", "-q", "--repo", repoUri, "--branch", branchName, .. policyParams ?? []]; + + await RunDarcAsync(commandParams); + } + + protected async Task GetRepositoryPolicies(string repoUri, string branchName) + { + return await RunDarcAsync("get-repository-policies", "--all", "--repo", repoUri, "--branch", branchName); + } + + protected async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, Octokit.PullRequest pr, Octokit.Repository repo, int attempts = 7) + { + while (attempts-- > 0) + { + TestContext.WriteLine($"Starting check for merge, attempts remaining {attempts}"); + pr = await GitHubApi.PullRequest.Get(repo.Id, pr.Number); + + if (pr.State == Octokit.ItemState.Closed) + { + return; + } + + await Task.Delay(TimeSpan.FromMinutes(1)); + } + + throw new ScenarioTestException($"The created pull request for {targetRepo} targeting {targetBranch} was not merged within {attempts} minutes"); + } + + protected async Task CheckGithubPullRequestChecks(string targetRepoName, string targetBranch) + { + TestContext.WriteLine($"Checking opened PR in {targetBranch} {targetRepoName}"); + Octokit.PullRequest pullRequest = await WaitForPullRequestAsync(targetRepoName, targetBranch); + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepoName); + + return await ValidateGithubMaestroCheckRunsSuccessful(targetRepoName, targetBranch, pullRequest, repo); + } + + protected async Task ValidateGithubMaestroCheckRunsSuccessful(string targetRepoName, string targetBranch, Octokit.PullRequest pullRequest, Octokit.Repository repo) + { + // Waiting 5 minutes 30 seconds for maestro to add the checks to the PR (it takes 5 minutes for the checks to be added) + await Task.Delay(TimeSpan.FromSeconds(5 * 60 + 30)); + TestContext.WriteLine($"Checking maestro merge policies check in {targetBranch} {targetRepoName}"); + Octokit.CheckRunsResponse existingCheckRuns = await GitHubApi.Check.Run.GetAllForReference(repo.Id, pullRequest.Head.Sha); + var cnt = 0; + foreach (var checkRun in existingCheckRuns.CheckRuns) + { + if (checkRun.ExternalId.StartsWith(MergePolicyConstants.MaestroMergePolicyCheckRunPrefix)) + { + cnt++; + if (checkRun.Status != "completed" && !checkRun.Output.Title.Contains("Waiting for checks.")) + { + TestContext.WriteLine($"Check '{checkRun.Output.Title}' with id {checkRun.Id} on PR {pullRequest.Url} has not completed in time. Check's status: {checkRun.Status}"); + return false; + } + } + } + + if (cnt == 0) + { + TestContext.WriteLine($"No maestro merge policy checks found in PR {pullRequest.Url}"); + return false; + } + + return true; + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs new file mode 100644 index 0000000000..63d7c3f95c --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs @@ -0,0 +1,262 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Microsoft.DotNet.DarcLib; +using NUnit.Framework; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Category("AzDO")] +[NonParallelizable] +internal class ScenarioTests_AzDoFlow : ScenarioTestBase +{ + private readonly IImmutableList _source1Assets; + private readonly IImmutableList _source2Assets; + private readonly IImmutableList _source1AssetsUpdated; + private readonly List _expectedAzDoDependenciesSource1; + private readonly List _expectedAzDoDependenciesSource2; + private readonly List _expectedAzDoDependenciesSource1Updated; + + public ScenarioTests_AzDoFlow() + { + _source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + _source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0"); + _source1AssetsUpdated = GetAssetData("Foo", "1.17.0", "Bar", "2.17.0"); + + var sourceRepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name); + var source2RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo3Name); + + _expectedAzDoDependenciesSource1 = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + ]; + + _expectedAzDoDependenciesSource2 = + [ + new DependencyDetail + { + Name = "Pizza", + Version = "3.1.0", + RepoUri = source2RepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Hamburger", + Version = "4.1.0", + RepoUri = source2RepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + ]; + + _expectedAzDoDependenciesSource1Updated = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + ]; + } + + [Test] + public async Task Darc_AzDoFlow_Batched() + { + TestContext.WriteLine("Azure DevOps Dependency Flow, batched"); + + TestParameters parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); + SetTestParameters(parameters); + var testLogic = new EndToEndFlowLogic(parameters); + var expectedDependencies = _expectedAzDoDependenciesSource1.Concat(_expectedAzDoDependenciesSource2).ToList(); + + await testLogic.DarcBatchedFlowTestBase( + $"AzDo_BatchedTestBranch_{Environment.MachineName}", + $"AzDo Batched Channel {Environment.MachineName}", + _source1Assets, + _source2Assets, + expectedDependencies, + true); + } + + [Test] + public async Task Darc_AzDoFlow_NonBatched_AllChecksSuccessful() + { + TestContext.WriteLine("AzDo Dependency Flow, non-batched, all checks successful"); + + TestParameters parameters = await TestParameters.GetAsync(); + SetTestParameters(parameters); + var testLogic = new EndToEndFlowLogic(parameters); + + await testLogic.NonBatchedAzDoFlowTestBase( + $"AzDo_NonBatchedTestBranch_AllChecks_{Environment.MachineName}", + $"AzDo Non-Batched All Checks Channel {Environment.MachineName}", + _source1Assets, + _expectedAzDoDependenciesSource1, + allChecks: true).ConfigureAwait(true); + } + + [Test] + public async Task Darc_AzDoFlow_NonBatched() + { + TestContext.WriteLine("AzDo Dependency Flow, non-batched"); + + TestParameters parameters = await TestParameters.GetAsync(); + SetTestParameters(parameters); + var testLogic = new EndToEndFlowLogic(parameters); + + await testLogic.NonBatchedUpdatingAzDoFlowTestBase( + $"AzDo_NonBatchedTestBranch_{Environment.MachineName}", + $"AzDo Non-Batched Channel {Environment.MachineName}", + _source1Assets, + _source1AssetsUpdated, + _expectedAzDoDependenciesSource1, + _expectedAzDoDependenciesSource1Updated).ConfigureAwait(false); + } + + [Test] + public async Task Darc_AzDoFlow_FeedFlow() + { + TestContext.WriteLine("AzDo Dependency Feed Flow, non-batched"); + + // Feed flow test strings + var proxyFeed = "https://some-proxy.azurewebsites.net/container/some-container/sig/somesig/se/2020-02-02/darc-int-maestro-test1-bababababab-1/index.json"; + var azdoFeed1 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-aaabaababababe-1/nuget/v3/index.json"; + var azdoFeed2 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-bbbbaababababd-1/nuget/v3/index.json"; + var azdoFeed3 = "https://some_org.pkgs.visualstudio.com/_packaging/darc-int-maestro-test1-cccbaababababf-1/nuget/v3/index.json"; + var regularFeed = "https://dotnetfeed.blob.core.windows.net/maestro-test1/index.json"; + var buildContainer = "https://dev.azure.com/dnceng/internal/_apis/build/builds/9999999/artifacts"; + string[] expectedFeeds = [proxyFeed, azdoFeed1, azdoFeed3]; + string[] notExpectedFeeds = [regularFeed, azdoFeed2, buildContainer]; + + IImmutableList feedFlowSourceAssets = ImmutableList.Create( + GetAssetDataWithLocations( + "Foo", + "1.1.0", + proxyFeed, + LocationType.NugetFeed + ), + GetAssetDataWithLocations( + "Bar", + "2.1.0", + azdoFeed1, + LocationType.NugetFeed), + GetAssetDataWithLocations( + "Pizza", + "3.1.0", + azdoFeed2, + LocationType.NugetFeed, + regularFeed, + LocationType.NugetFeed + ), + GetAssetDataWithLocations( + "Hamburger", + "4.1.0", + azdoFeed3, + LocationType.NugetFeed, + buildContainer, + LocationType.Container) + ); + + TestContext.WriteLine("Azure DevOps Internal feed flow"); + TestParameters parameters = await TestParameters.GetAsync(); + SetTestParameters(parameters); + + var testLogic = new EndToEndFlowLogic(parameters); + + var expectedAzDoFeedFlowDependencies = new List(); + + var feedFoo = new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false, + Locations = new List { proxyFeed } + }; + expectedAzDoFeedFlowDependencies.Add(feedFoo); + + var feedBar = new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false, + Locations = new List { azdoFeed1 } + }; + expectedAzDoFeedFlowDependencies.Add(feedBar); + + var feedPizza = new DependencyDetail + { + Name = "Pizza", + Version = "3.1.0", + RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false, + Locations = new List { azdoFeed2, regularFeed } + }; + expectedAzDoFeedFlowDependencies.Add(feedPizza); + + var feedHamburger = new DependencyDetail + { + Name = "Hamburger", + Version = "4.1.0", + RepoUri = GetAzDoRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false, + Locations = new List { azdoFeed3, buildContainer } + }; + expectedAzDoFeedFlowDependencies.Add(feedHamburger); + await testLogic.NonBatchedAzDoFlowTestBase( + $"AzDo_FeedFlowBranch_{Environment.MachineName}", + $"AzDo_FeedFlowChannel_{Environment.MachineName}", + feedFlowSourceAssets, + expectedAzDoFeedFlowDependencies, + isFeedTest: true, + expectedFeeds: expectedFeeds, + notExpectedFeeds: notExpectedFeeds).ConfigureAwait(false); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Builds.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Builds.cs new file mode 100644 index 0000000000..f9e26d37a3 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Builds.cs @@ -0,0 +1,106 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using FluentAssertions; +using NUnit.Framework; +using NUnit.Framework.Internal; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Parallelizable] +internal class ScenarioTests_Builds : ScenarioTestBase +{ + private string _repoUrl; + private readonly string _repoName = TestRepository.TestRepo1Name; + private const string SourceBuildNumber = "654321"; + private const string SourceCommit = "123456"; + private const string SourceBranch = "master"; + + private readonly IImmutableList _sourceAssets; + private TestParameters _parameters; + + public ScenarioTests_Builds() + { + _sourceAssets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + } + + [SetUp] + public async Task InitializeAsync() + { + _parameters = await TestParameters.GetAsync(); + SetTestParameters(_parameters); + } + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + // Create a new build and check some of the metadata. Then mark as released and check again + [Test] + public async Task ArcadeBuilds_EndToEnd() + { + TestContext.WriteLine("Darc/Maestro build-handling tests"); + var scenarioDirectory = _parameters._dir.Directory; + + _repoUrl = GetGitHubRepoUrl(_repoName); + + // Create a build for the source repo + Build build = await CreateBuildAsync(_repoUrl, SourceBranch, SourceCommit, SourceBuildNumber, _sourceAssets); + Build retrievedBuild = await PcsApi.Builds.GetBuildAsync(build.Id); + retrievedBuild.Released.Should().BeFalse("Retrieved build has Released set to true when it should be false"); + + // Release the build; gather-drop does not fetch anything unless the flag '--include-released' is included in its arguments (which the next operation does set) + Build updatedBuild = await PcsApi.Builds.UpdateAsync(new BuildUpdate() { Released = true }, build.Id); + updatedBuild.Released.Should().BeTrue("Retrieved build has Released set to false when it should be true"); + + // Gather a drop with release included + var gatherWithReleasedDir = Path.Combine(scenarioDirectory, "gather-with-released"); + var gatherDropOutput = ""; + + TestContext.WriteLine("Starting 'Gather with released, where build has been set to released' using folder " + gatherWithReleasedDir); + + gatherDropOutput = await GatherDrop(build.Id, gatherWithReleasedDir, true, string.Empty); + + gatherDropOutput.Should().Contain($"Gathering drop for build {SourceBuildNumber}", "Gather with released 1"); + gatherDropOutput.Should().Contain("Downloading asset Bar@2.1.0", "Gather with released 1"); + gatherDropOutput.Should().Contain("Downloading asset Foo@1.1.0", "Gather with released 1"); + gatherDropOutput.Should().NotContain("always-download assets in build", "Gather with released 1"); + + // Gather with release excluded (default behavior). Gather-drop should throw an error. + TestContext.WriteLine("Starting 'Gather with release excluded' - gather-drop should throw an error."); + + var gatherWithNoReleasedDir = Path.Combine(scenarioDirectory, "gather-no-released"); + Assert.ThrowsAsync(async () => await GatherDrop(build.Id, gatherWithNoReleasedDir, false, string.Empty), "Gather with release excluded"); + + // Unrelease the build + Build unreleaseBuild = await PcsApi.Builds.UpdateAsync(new BuildUpdate() { Released = false }, build.Id); + unreleaseBuild.Released.Should().BeFalse(); + + // Gather with release excluded again (default behavior) + var gatherWithNoReleased2Dir = Path.Combine(scenarioDirectory, "gather-no-released-2"); + TestContext.WriteLine("Starting 'Gather unreleased with release excluded' using folder " + gatherWithNoReleased2Dir); + gatherDropOutput = await GatherDrop(build.Id, gatherWithNoReleased2Dir, false, string.Empty); + + gatherDropOutput.Should().Contain($"Gathering drop for build {SourceBuildNumber}", "Gather unreleased with release excluded"); + gatherDropOutput.Should().Contain("Downloading asset Bar@2.1.0", "Gather unreleased with release excluded"); + gatherDropOutput.Should().Contain("Downloading asset Foo@1.1.0", "Gather unreleased with release excluded"); + gatherDropOutput.Should().NotContain("always-download assets in build", "Gather unreleased with release excluded"); + + // Gather with release excluded again, but specify --always-download-asset-filters + var gatherWithNoReleased3Dir = Path.Combine(scenarioDirectory, "gather-no-released-3"); + TestContext.WriteLine("Starting 'Gather unreleased with release excluded' using folder " + gatherWithNoReleased3Dir); + gatherDropOutput = await GatherDrop(build.Id, gatherWithNoReleased3Dir, false, $"B.*r$"); + + gatherDropOutput.Should().Contain($"Gathering drop for build {SourceBuildNumber}", "Gather unreleased with release excluded"); + gatherDropOutput.Should().Contain("Downloading asset Bar@2.1.0", "Gather unreleased with release excluded"); + gatherDropOutput.Should().Contain("Downloading asset Foo@1.1.0", "Gather unreleased with release excluded"); + gatherDropOutput.Should().Contain("Found 1 always-download asset(s) in build", "Gather unreleased with release excluded"); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs new file mode 100644 index 0000000000..be36407b60 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using NUnit.Framework; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Parallelizable] +internal class ScenarioTests_Channels : ScenarioTestBase +{ + private TestParameters _parameters; + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [Test] + public async Task ArcadeChannels_EndToEnd() + { + _parameters = await TestParameters.GetAsync(); + SetTestParameters(_parameters); + + // Create a new channel + var testChannelName = $"Test Channel End to End {Environment.MachineName}"; + + await using (AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false)) + { + // Get the channel and make sure it's there + var returnedChannel = await GetTestChannelsAsync().ConfigureAwait(false); + returnedChannel.Should().Contain(testChannelName, "Channel was not created or could not be retrieved"); + + // Delete the channel + await DeleteTestChannelAsync(testChannelName).ConfigureAwait(false); + + // Get the channel and make sure it was deleted + var returnedChannel2 = await GetTestChannelsAsync().ConfigureAwait(false); + returnedChannel2.Should().NotContain(testChannelName, "Channel was not deleted"); + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Clone.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Clone.cs new file mode 100644 index 0000000000..33e5283426 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Clone.cs @@ -0,0 +1,202 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Security.Cryptography; +using FluentAssertions; +using NUnit.Framework; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[NonParallelizable] +[Category("PostDeployment")] +internal class ScenarioTests_Clone : ScenarioTestBase +{ + [Test] + [Ignore("We no longer use this functionality")] + public async Task Darc_CloneRepo() + { + TestContext.WriteLine("Darc-Clone repo end to end test"); + + TestParameters parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); + SetTestParameters(parameters); + + var sourceRepoName = "core-sdk"; + var sourceRepoVersion = "v3.0.100-preview4-011223"; + var sourceRepoUri = GetGitHubRepoUrl(sourceRepoName); + + // these repos are not currently clonable for us due to auth + var reposToIgnore = "https://dev.azure.com/dnceng/internal/_git/dotnet-optimization;https://dev.azure.com/devdiv/DevDiv/_git/DotNet-Trusted;https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Trusted"; + + // these repos have file names that are too long on Windows for the temp folder + reposToIgnore += ";https://github.com/aspnet/AspNetCore;https://github.com/aspnet/AspNetCore-Tooling;https://github.com/dotnet/core-setup;https://github.com/dotnet/templating;" + + "https://github.com/dotnet/sdk;https://github.com/Microsoft/visualfsharp;https://github.com/dotnet/roslyn;https://github.com/NuGet/NuGet.Client;https://github.com/dotnet/corefx"; + + var expectedRepos1 = new Dictionary + { + { "cli.204f425b6f061d0b8a01faf46f762ecf71436f68", "BB7B735A345157E1CB90E4E03340FD202C798B06E5EA089C2F0191562D9DF1B4" }, + { "cliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01", "CCA267C3FB69E8AADFA6CEE474249ACB84829D9CCFE7F9F5D7F17EFA87A26739" }, + { "core-sdk.v3.0.100-preview4-011223", "BBAE24214F40518798126A2F4729C38B6C3B67886DCDE0C2D64BBE203D4ACFBB" }, + { "msbuild.d004974104fde202e633b3c97e0ece3287aa62f9", "FD4F85D3FB60B5EBC105647E290924071A240A0C6FD49E3D44A2BC38A797946C" }, + { "standard.8ef5ada20b5343b0cb9e7fd577341426dab76cd8", "FA876A709FDB0CD3D864431B2887D2BC9ACBFC313E8C452A8636EE7FAE4E5D02" }, + { "toolset.3165b2579582b6b44aef768c01c3cc9ff4f0bc17", "D483493BF79129FD2DADC39C51FACB5FA363CEB03E54ECD180720DE2D95EDED9" }, + { "websdk.b55d4f4cf22bee7ec9a2ca5f49d54ebf6ee67e83", "A51DEB15209039D17FC0D781BFC445F8C5CDC1D51673AC4C0929FBEE8C1E4D21" }, + { "winforms.b1ee29b8b8e14c1200adff02847391dde471d0d2", "1288479D12130F9AEF56137DCC655B5B59277F5153172B30DC419456AF9CA011" }, + { "wpf.d378b1ec6b8555c52b7da1c40ffc0784cb0f5cad", "CFE5E19366FD5A90101A8F369CFC70611F9B1439A0720D435EE34872AF55A40A" } + }; + + string[] expectedMasterRepos1 = + [ + "cli", + "cliCommandLineParser", + "core-sdk", + "msbuild", + "standard", + "toolset", + "websdk", + "winforms", + "wpf" + ]; + + string[] expectedGitDirs1 = + [ + "cli.git", + "cliCommandLineParser.git", + "core-sdk.git", + "msbuild.git", + "standard.git", + "toolset.git", + "websdk.git", + "winforms.git", + "wpf.git" + ]; + + TestContext.WriteLine($"parameters: sourceRepoName={sourceRepoName}, sourceRepoVersion={sourceRepoVersion}, reposToIgnore='{reposToIgnore}'"); + TestContext.WriteLine($"Cloning repo {sourceRepoUri} at {sourceRepoVersion} with depth 2 and include - toolset = false"); + TemporaryDirectory reposFolder = await CloneRepositoryWithDarc(sourceRepoName, sourceRepoVersion, reposToIgnore, false, 2); + CheckExpectedClonedRepos(expectedRepos1, expectedMasterRepos1, expectedGitDirs1, reposFolder); + + var expectedRepos2 = new Dictionary + { + { "cli.204f425b6f061d0b8a01faf46f762ecf71436f68", "BB7B735A345157E1CB90E4E03340FD202C798B06E5EA089C2F0191562D9DF1B4" }, + { "cliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01", "CCA267C3FB69E8AADFA6CEE474249ACB84829D9CCFE7F9F5D7F17EFA87A26739" }, + { "core-sdk.v3.0.100-preview4-011223", "BBAE24214F40518798126A2F4729C38B6C3B67886DCDE0C2D64BBE203D4ACFBB" }, + { "coreclr.d833cacabd67150fe3a2405845429a0ba1b72c12", "5A69F61C354C1DE9B839D6922CE866AB0B760BCCBC919EE4C408D44B6A40084C" }, + { "msbuild.d004974104fde202e633b3c97e0ece3287aa62f9", "FD4F85D3FB60B5EBC105647E290924071A240A0C6FD49E3D44A2BC38A797946C" }, + { "standard.8ef5ada20b5343b0cb9e7fd577341426dab76cd8", "FA876A709FDB0CD3D864431B2887D2BC9ACBFC313E8C452A8636EE7FAE4E5D02" }, + { "toolset.3165b2579582b6b44aef768c01c3cc9ff4f0bc17", "D483493BF79129FD2DADC39C51FACB5FA363CEB03E54ECD180720DE2D95EDED9" }, + { "websdk.b55d4f4cf22bee7ec9a2ca5f49d54ebf6ee67e83", "A51DEB15209039D17FC0D781BFC445F8C5CDC1D51673AC4C0929FBEE8C1E4D21" }, + { "winforms.b1ee29b8b8e14c1200adff02847391dde471d0d2", "1288479D12130F9AEF56137DCC655B5B59277F5153172B30DC419456AF9CA011" }, + {"wpf.d378b1ec6b8555c52b7da1c40ffc0784cb0f5cad", "CFE5E19366FD5A90101A8F369CFC70611F9B1439A0720D435EE34872AF55A40A" } + }; + + string[] expectedMasterRepos2 = + [ + "cli", + "cliCommandLineParser", + "core-sdk", + "coreclr", + "msbuild", + "standard", + "toolset", + "websdk", + "winforms", + "wpf" + ]; + + string[] expectedGitDirs2 = + [ + "cli.git", + "cliCommandLineParser.git", + "core-sdk.git", + "coreclr.git", + "msbuild.git", + "standard.git", + "toolset.git", + "websdk.git", + "winforms.git", + "wpf.git" + ]; + + reposToIgnore += ";https://github.com/dotnet/arcade"; + TestContext.WriteLine($"Cloning repo {sourceRepoUri} at {sourceRepoVersion} with depth 4 and include-toolset=true"); + reposFolder = await CloneRepositoryWithDarc(sourceRepoName, sourceRepoVersion, reposToIgnore, true, 4); + CheckExpectedClonedRepos(expectedRepos2, expectedMasterRepos2, expectedGitDirs2, reposFolder); + } + + private static void CheckExpectedClonedRepos(Dictionary expectedRepos, string[] expectedMasterRepos, string[] expectedGitDirs, TemporaryDirectory reposFolder) + { + using (ChangeDirectory(reposFolder.Directory)) + { + var clonedReposFolder = Path.Join(reposFolder.Directory, "cloned-repos"); + var gitDirFolder = Path.Join(reposFolder.Directory, "git-dirs"); + + TestContext.WriteLine("Validate the hash of the Version.Details.xml matches expected"); + foreach (var name in expectedRepos.Keys) + { + var path = Path.Join(clonedReposFolder, name); + Directory.Exists(path).Should().BeTrue($"Expected cloned repo '{name}' but not found at {path}"); + + var versionPath = Path.Join(path, "eng", "Version.Details.xml"); + File.Exists(versionPath).Should().BeTrue($"Expected a file at {versionPath}"); + + using (FileStream stream = File.OpenRead(versionPath)) + { + using var hashGenerator = SHA256.Create(); + var fileHash = hashGenerator.ComputeHash(stream); + var fileHashInHex = BitConverter.ToString(fileHash).Replace("-", ""); + fileHashInHex.Should().Be(expectedRepos[name], $"Expected {versionPath} to have hash '{expectedRepos[name]}', actual hash '{fileHash}'"); + } + } + + TestContext.WriteLine("Ensure the presence of the git directories"); + foreach (var repo in expectedMasterRepos) + { + var path = Path.Join(clonedReposFolder, repo); + Directory.Exists(path).Should().BeTrue($"Expected cloned master repo {repo} but it is missing"); + + var gitRedirectPath = Path.Join(path, ".git"); + var expectedGitDir = Path.Join(gitDirFolder, repo); + var expectedRedirect = $"gitdir: {expectedGitDir}.git"; + var actualRedirect = File.ReadAllText(gitRedirectPath); + actualRedirect.Should().Be(expectedRedirect, $"Expected {path} to have .gitdir redirect of {expectedRedirect}, actual {actualRedirect}"); + } + + TestContext.WriteLine("Ensure the presence of the cloned repo directories"); + var allRepos = Directory.GetDirectories(clonedReposFolder); + IEnumerable reposFolderNames = allRepos.Select(Path.GetFileName); + var actualRepos = reposFolderNames.Where(fn => fn.Contains('.')).ToList(); + var actualMasterRepos = reposFolderNames.Where(fn => !fn.Contains('.')).ToList(); + + actualRepos.Should().HaveCount(expectedRepos.Count); + actualRepos.Should().BeEquivalentTo(expectedRepos.Keys); + + actualMasterRepos.Should().HaveCount(expectedMasterRepos.Length); + actualMasterRepos.Should().BeEquivalentTo(expectedMasterRepos); + + TestContext.WriteLine("Validating the existence and content of the git directories"); + foreach (var dir in expectedGitDirs) + { + var path = Path.Join(gitDirFolder, dir); + Directory.Exists(path).Should().BeTrue($"Expected a .gitdir for '{dir}', but not found at {path}"); + } + + var actualGitDirs = Directory.GetDirectories(gitDirFolder); + var actualGitDirNames = actualGitDirs.Select(Path.GetFileName).ToList(); + actualGitDirNames.Should().BeEquivalentTo(expectedGitDirs); + + foreach (var gitDirectory in actualGitDirs) + { + TestContext.WriteLine($"Checking content of {gitDirectory}"); + string[] expectedFolders = ["hooks", "info", "logs", "objects", "refs"]; + var actualFolders = Directory.GetDirectories(gitDirectory); + var actualFolderNames = actualFolders.Select(Path.GetFileName).ToList(); + actualFolderNames.Should().BeEquivalentTo(expectedFolders); + + string[] expectedFiles = ["config", "description", "FETCH_HEAD", "HEAD", "index"]; + var actualFiles = Directory.GetFiles(gitDirectory).Select(Path.GetFileName).ToArray(); + actualFiles.Should().BeEquivalentTo(expectedFiles); + } + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs new file mode 100644 index 0000000000..83689de3e9 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using NUnit.Framework; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Parallelizable] +internal class ScenarioTests_DefaultChannels : ScenarioTestBase +{ + private readonly string _repoName = TestRepository.TestRepo1Name; + private readonly string _branchName; + private readonly string _branchNameWithRefsHeads; + private TestParameters _parameters; + + public ScenarioTests_DefaultChannels() + { + _branchName = $"ChannelTestBranch_{Environment.MachineName}"; + _branchNameWithRefsHeads = $"refs/heads/{_branchName}"; + } + + [SetUp] + public async Task InitializeAsync() + { + _parameters = await TestParameters.GetAsync(); + SetTestParameters(_parameters); + } + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [Test] + public async Task ArcadeChannels_DefaultChannels() + { + var repoUrl = GetGitHubRepoUrl(_repoName); + + var testChannelName1 = $"TestChannelDefault1_{Environment.MachineName}"; + var testChannelName2 = $"TestChannelDefault2_{Environment.MachineName}"; + + await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(testChannelName1)) + { + await using (AsyncDisposableValue channel2 = await CreateTestChannelAsync(testChannelName2)) + { + await AddDefaultTestChannelAsync(testChannelName1, repoUrl, _branchNameWithRefsHeads); + await AddDefaultTestChannelAsync(testChannelName2, repoUrl, _branchNameWithRefsHeads); + + var defaultChannels = await GetDefaultTestChannelsAsync(repoUrl, _branchName); + defaultChannels.Should().Contain(testChannelName1, $"{testChannelName1} is not a default channel"); + defaultChannels.Should().Contain(testChannelName2, $"{testChannelName2} is not a default channel"); + + await DeleteDefaultTestChannelAsync(testChannelName1, repoUrl, _branchName); + await DeleteDefaultTestChannelAsync(testChannelName2, repoUrl, _branchName); + + defaultChannels = await GetDefaultTestChannelsAsync(repoUrl, _branchName); + defaultChannels.Should().NotContain(testChannelName1, $"{testChannelName1} was not deleted from default channels"); + defaultChannels.Should().NotContain(testChannelName2, $"{testChannelName2} was not deleted from default channels"); + } + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs new file mode 100644 index 0000000000..2014d20a82 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs @@ -0,0 +1,96 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using FluentAssertions; +using NUnit.Framework; +using NUnit.Framework.Internal; +using NUnit.Framework.Legacy; +using ProductConstructionService.Client.Models; +using ProductConstructionService.ScenarioTests.ObjectHelpers; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Category("GitHub")] +[Parallelizable] +internal class ScenarioTests_Dependencies : ScenarioTestBase +{ + + private TestParameters _parameters; + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [Test] + public async Task ArcadeDependencies_EndToEnd() + { + _parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); + SetTestParameters(_parameters); + + var source1RepoName = TestRepository.TestRepo1Name; + var source2RepoName = TestRepository.TestRepo3Name; + var targetRepoName = TestRepository.TestRepo2Name; + var target1BuildNumber = "098765"; + var target2BuildNumber = "987654"; + var sourceBuildNumber = "654321"; + var sourceCommit = "SourceCommitVar"; + var targetCommit = "TargetCommitVar"; + var sourceBranch = $"DependenciesSourceBranch_{Environment.MachineName}"; + var targetBranch = $"DependenciesTargetBranch_{Environment.MachineName}"; + var testChannelName = $"TestChannel_Dependencies_{Environment.MachineName}"; + + IImmutableList source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + IImmutableList source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0"); + IImmutableList targetAssets = GetAssetData("Source1", "3.1.0", "Source2", "4.1.0"); + var source1RepoUri = GetGitHubRepoUrl(source1RepoName); + var source2RepoUri = GetGitHubRepoUrl(source2RepoName); + var targetRepoUri = GetGitHubRepoUrl(targetRepoName); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await CreateTestChannelAsync(testChannelName); + + TestContext.WriteLine("Set up build1 for intake into target repository"); + Build build1 = await CreateBuildAsync(source1RepoUri, sourceBranch, sourceCommit, sourceBuildNumber, source1Assets); + await AddBuildToChannelAsync(build1.Id, testChannelName); + + TestContext.WriteLine("Set up build2 for intake into target repository"); + Build build2 = await CreateBuildAsync(source2RepoUri, sourceBranch, sourceCommit, sourceBuildNumber, source2Assets); + await AddBuildToChannelAsync(build2.Id, testChannelName); + + ImmutableList dependencies = ImmutableList.Empty; + var buildRef1 = new BuildRef(build1.Id, true, 1); + dependencies = dependencies.Add(buildRef1); + var buildRef2 = new BuildRef(build2.Id, true, 2); + dependencies = dependencies.Add(buildRef2); + + // Add the target build once, should populate the BuildDependencies table and calculate TimeToInclusion + TestContext.WriteLine("Set up targetBuild in target repository"); + Build targetBuild1 = await CreateBuildAsync(targetRepoUri, targetBranch, targetCommit, target1BuildNumber, targetAssets, dependencies); + await AddBuildToChannelAsync(targetBuild1.Id, testChannelName); + var newTargetBuild1 = new Build(targetBuild1.Id, targetBuild1.DateProduced, targetBuild1.Staleness, targetBuild1.Released, + targetBuild1.Stable, targetBuild1.Commit, targetBuild1.Channels, targetBuild1.Assets, dependencies, incoherencies: null); + + // Add the target build a second time, should populate the BuildDependencies table and use the previous TimeToInclusion + Build targetBuild2 = await CreateBuildAsync(targetRepoUri, targetBranch, targetCommit, target2BuildNumber, targetAssets, dependencies); + await AddBuildToChannelAsync(targetBuild2.Id, testChannelName); + var newTargetBuild2 = new Build(targetBuild2.Id, targetBuild2.DateProduced, targetBuild2.Staleness, targetBuild2.Released, + targetBuild2.Stable, targetBuild2.Commit, targetBuild2.Channels, targetBuild2.Assets, dependencies, incoherencies: null); + + Build retrievedBuild1 = await PcsApi.Builds.GetBuildAsync(targetBuild1.Id); + Build retrievedBuild2 = await PcsApi.Builds.GetBuildAsync(targetBuild2.Id); + + retrievedBuild1.Dependencies.Should().HaveCount(newTargetBuild1.Dependencies.Count); + retrievedBuild2.Dependencies.Should().HaveCount(newTargetBuild2.Dependencies.Count); + + var buildRefComparer = new BuildRefComparer(); + + CollectionAssert.AreEqual(retrievedBuild1.Dependencies, newTargetBuild1.Dependencies, buildRefComparer); + CollectionAssert.AreEqual(retrievedBuild2.Dependencies, newTargetBuild2.Dependencies, buildRefComparer); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs new file mode 100644 index 0000000000..4d6e60ded9 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs @@ -0,0 +1,318 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using Microsoft.DotNet.DarcLib; +using NUnit.Framework; +using NUnit.Framework.Internal; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Category("GitHub")] +[Parallelizable] +internal class ScenarioTests_GitHubFlow : ScenarioTestBase +{ + private readonly IImmutableList _source1Assets; + private readonly IImmutableList _source2Assets; + private readonly IImmutableList _source1AssetsUpdated; + private readonly List _expectedDependenciesSource1; + private readonly List _expectedDependenciesSource2; + private readonly List _expectedDependenciesSource1Updated; + + public ScenarioTests_GitHubFlow() + { + using TestParameters parameters = TestParameters.GetAsync().Result; + SetTestParameters(parameters); + + _source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + _source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0"); + _source1AssetsUpdated = GetAssetData("Foo", "1.17.0", "Bar", "2.17.0"); + + var sourceRepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name); + var source2RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo3Name); + + _expectedDependenciesSource1 = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + } + ]; + + _expectedDependenciesSource2 = + [ + new DependencyDetail + { + Name = "Pizza", + Version = "3.1.0", + RepoUri = source2RepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Hamburger", + Version = "4.1.0", + RepoUri = source2RepoUri, + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + } + ]; + + _expectedDependenciesSource1Updated = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.17.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo2Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.17.0", + RepoUri = sourceRepoUri, + Commit = TestRepository.CoherencyTestRepo2Commit, + Type = DependencyType.Product, + Pinned = false + } + ]; + } + + [Test] + public async Task Darc_GitHubFlow_Batched() + { + TestContext.WriteLine("Github Dependency Flow, batched"); + + using TestParameters parameters = await TestParameters.GetAsync(); + var testLogic = new EndToEndFlowLogic(parameters); + var expectedDependencies = _expectedDependenciesSource1.Concat(_expectedDependenciesSource2).ToList(); + + await testLogic.DarcBatchedFlowTestBase( + $"GitHub_BatchedTestBranch_{Environment.MachineName}", + $"GitHub Batched Channel {Environment.MachineName}", + _source1Assets, + _source2Assets, + expectedDependencies, + false).ConfigureAwait(false); + } + + [Test] + public async Task Darc_GitHubFlow_NonBatched() + { + TestContext.WriteLine("GitHub Dependency Flow, non-batched"); + + using TestParameters parameters = await TestParameters.GetAsync(); + var testLogic = new EndToEndFlowLogic(parameters); + + await testLogic.NonBatchedUpdatingGitHubFlowTestBase( + $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", + $"GitHub Non-Batched Channel {Environment.MachineName}", + _source1Assets, + _source1AssetsUpdated, + _expectedDependenciesSource1, + _expectedDependenciesSource1Updated).ConfigureAwait(false); + } + + [Test] + public async Task Darc_GitHubFlow_NonBatched_StrictCoherency() + { + TestContext.WriteLine("GitHub Dependency Flow, non-batched"); + + using TestParameters parameters = await TestParameters.GetAsync(); + var testLogic = new EndToEndFlowLogic(parameters); + + List expectedCoherencyDependencies = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + } + ]; + + IImmutableList sourceAssets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + + await testLogic.NonBatchedGitHubFlowTestBase( + $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", + $"GitHub Non-Batched Channel {Environment.MachineName}", + sourceAssets, + expectedCoherencyDependencies, + allChecks: true).ConfigureAwait(false); + } + + [Test] + public async Task Darc_GitHubFlow_NonBatched_FailingCoherencyUpdate() + { + using TestParameters parameters = await TestParameters.GetAsync(); + var testLogic = new EndToEndFlowLogic(parameters); + + List expectedCoherencyDependencies = + [ + new DependencyDetail + { + Name = "Foo", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Bar", + Version = "2.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "Fzz", + Version = "", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = "", + Type = DependencyType.Product, + CoherentParentDependencyName = "Foo" + }, + new DependencyDetail + { + Name = "ASD", + Version = "", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = "", + Type = DependencyType.Product, + CoherentParentDependencyName = "Foo" + }, + ]; + + IImmutableList sourceAssets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); + IImmutableList childSourceAssets = GetAssetData("Fzz", "1.1.0", "ASD", "1.1.1"); + + await testLogic.NonBatchedGitHubFlowCoherencyTestBase( + $"GitHub_NonBatchedTestBranch_FailingCoherencyUpdate_{Environment.MachineName}", + $"GitHub Non-Batched Channel FailingCoherencyUpdate {Environment.MachineName}", + sourceAssets, + childSourceAssets, + expectedCoherencyDependencies, + coherentParent: "Foo", + allChecks: false).ConfigureAwait(false); + } + + [Test] + public async Task Darc_GitHubFlow_NonBatched_FailingCoherentOnlyUpdate() + { + using TestParameters parameters = await TestParameters.GetAsync(); + var testLogic = new EndToEndFlowLogic(parameters); + + List expectedNonCoherencyDependencies = + [ + new DependencyDetail + { + Name = "A1", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "A2", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + } + ]; + + List expectedCoherencyDependencies = + [ + new DependencyDetail + { + Name = "A1", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "A2", + Version = "1.1.0", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo2Name), + Commit = TestRepository.CoherencyTestRepo1Commit, + Type = DependencyType.Product, + Pinned = false + }, + new DependencyDetail + { + Name = "B1", + Version = "", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = "", + Type = DependencyType.Product, + CoherentParentDependencyName = "A1" + }, + new DependencyDetail + { + Name = "B2", + Version = "", + RepoUri = GetGitHubRepoUrl(TestRepository.TestRepo1Name), + Commit = "", + Type = DependencyType.Product, + CoherentParentDependencyName = "A1" + }, + ]; + + IImmutableList sourceAssets = GetAssetData("A1", "1.1.0", "A2", "1.1.0"); + IImmutableList childSourceAssets = GetAssetData("B1", "2.1.0", "B2", "2.1.0"); + + await testLogic.NonBatchedGitHubFlowCoherencyOnlyTestBase( + $"GitHub_NonBatchedTestBranch_FailingCoherencyOnlyUpdate_{Environment.MachineName}", + $"GitHub Non-Batched Channel FailingCoherencyOnlyUpdate {Environment.MachineName}", + sourceAssets, + childSourceAssets, + expectedNonCoherencyDependencies, + expectedCoherencyDependencies, + coherentParent: "A1").ConfigureAwait(false); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_MergePolicies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_MergePolicies.cs new file mode 100644 index 0000000000..6751f23a6c --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_MergePolicies.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using FluentAssertions; +using NUnit.Framework; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Parallelizable] +internal class ScenarioTests_MergePolicies : ScenarioTestBase +{ + private TestParameters _parameters; + private readonly Random _random = new(); + private const string SourceRepo = "maestro-test1"; + private const string TargetRepo = "maestro-test2"; + + [SetUp] + public async Task InitializeAsync() + { + _parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); + SetTestParameters(_parameters); + } + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + private string GetTestChannelName() + { + return "Test Channel " + _random.Next(int.MaxValue); + } + + private string GetTargetBranch() + { + return Guid.NewGuid().ToString(); + } + + [Test] + public async Task Darc_GitHubFlow_AutoMerge_GithubChecks_AllChecksSuccessful() + { + var testChannelName = GetTestChannelName(); + var targetBranch = GetTargetBranch(); + + await AutoMergeFlowTestBase(TargetRepo, SourceRepo, targetBranch, testChannelName, ["--all-checks-passed"]); + } + + [Test] + public async Task Darc_GitHubFlow_AutoMerge_GithubChecks_ValidateCoherencyCheck() + { + var testChannelName = GetTestChannelName(); + var targetBranch = GetTargetBranch(); + + await AutoMergeFlowTestBase(TargetRepo, SourceRepo, targetBranch, testChannelName, ["--validate-coherency"]); + } + + [Test] + public async Task Darc_GitHubFlow_AutoMerge_GithubChecks_Standard() + { + var testChannelName = GetTestChannelName(); + var targetBranch = GetTargetBranch(); + + await AutoMergeFlowTestBase(TargetRepo, SourceRepo, targetBranch, testChannelName, ["--standard-automerge"]); + } + + [Test] + public async Task Darc_GitHubFlow_AutoMerge_GithubChecks_NoRequestedChanges() + { + var testChannelName = GetTestChannelName(); + var targetBranch = GetTargetBranch(); + + await AutoMergeFlowTestBase(TargetRepo, SourceRepo, targetBranch, testChannelName, ["--no-requested-changes"]); + } + + public async Task AutoMergeFlowTestBase(string targetRepo, string sourceRepo, string targetBranch, string testChannelName, List args) + { + var targetRepoUri = GetGitHubRepoUrl(targetRepo); + var sourceRepoUri = GetGitHubRepoUrl(sourceRepo); + var sourceBranch = "dependencyflow-tests"; + var sourceCommit = "0b36b99e29b1751403e23cfad0a7dff585818051"; + var sourceBuildNumber = _random.Next(int.MaxValue).ToString(); + ImmutableList sourceAssets = ImmutableList.Create() + .Add(new AssetData(true) + { + Name = "Foo", + Version = "1.1.0", + }) + .Add(new AssetData(true) + { + Name = "Bar", + Version = "2.1.0", + }); + + TestContext.WriteLine($"Creating test channel {testChannelName}"); + await using AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName); + + TestContext.WriteLine($"Adding a subscription from ${sourceRepo} to ${targetRepo}"); + await using AsyncDisposableValue sub = await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", "maestro-auth-test", additionalOptions: args); + + TestContext.WriteLine("Set up build for intake into target repository"); + var build = await CreateBuildAsync(sourceRepoUri, sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); + await using IAsyncDisposable _ = await AddBuildToChannelAsync(build.Id, testChannelName); + + TestContext.WriteLine("Cloning target repo to prepare the target branch"); + using TemporaryDirectory repo = await CloneRepositoryAsync(targetRepo); + using (ChangeDirectory(repo.Directory)) + { + await RunGitAsync("checkout", "-b", targetBranch); + TestContext.WriteLine("Adding dependencies to target repo"); + await AddDependenciesToLocalRepo(repo.Directory, "Foo", sourceRepoUri); + await AddDependenciesToLocalRepo(repo.Directory, "Bar", sourceRepoUri); + + TestContext.WriteLine("Pushing branch to remote"); + await RunGitAsync("commit", "-am", "Add dependencies."); + await using IAsyncDisposable ___ = await PushGitBranchAsync("origin", targetBranch); + + await TriggerSubscriptionAsync(sub.Value); + + TestContext.WriteLine($"Waiting on PR to be opened in ${targetRepoUri}"); + var testResult = await CheckGithubPullRequestChecks(targetRepo, targetBranch); + testResult.Should().BeTrue(); + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs new file mode 100644 index 0000000000..80c05bc637 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using NUnit.Framework; +using NUnit.Framework.Internal; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Parallelizable] +internal class ScenarioTests_RepoPolicies : ScenarioTestBase +{ + // The RepoPolicies logic does a partial string match for the branch name in the base, + // so it's important that this branch name not be a substring or superstring of another branch name + private readonly string _branchName = $"MaestroRepoPoliciesTestBranch_{Environment.MachineName}"; + private readonly string _repoName = TestRepository.TestRepo1Name; + private TestParameters _parameters; + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [Test] + public async Task ArcadeRepoPolicies_EndToEnd() + { + TestContext.WriteLine("Repository merge policy handling"); + TestContext.WriteLine("Running tests..."); + + _parameters = await TestParameters.GetAsync(); + SetTestParameters(_parameters); + + var repoUrl = GetGitHubRepoUrl(_repoName); + + TestContext.WriteLine("Setting repository merge policy to empty"); + await SetRepositoryPolicies(repoUrl, _branchName); + var emptyPolicies = await GetRepositoryPolicies(repoUrl, _branchName); + var expectedEmpty = $"{repoUrl} @ {_branchName}\r\n- Merge Policies: []\r\n"; + emptyPolicies.Should().BeEquivalentTo(expectedEmpty, "Repository merge policy is not empty"); + + TestContext.WriteLine("Setting repository merge policy to standard"); + await SetRepositoryPolicies(repoUrl, _branchName, ["--standard-automerge"]); + var standardPolicies = await GetRepositoryPolicies(repoUrl, _branchName); + var expectedStandard = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n Standard\r\n"; + standardPolicies.Should().BeEquivalentTo(expectedStandard, "Repository policy not set to standard"); + + TestContext.WriteLine("Setting repository merge policy to all checks successful"); + await SetRepositoryPolicies(repoUrl, _branchName, ["--all-checks-passed", "--ignore-checks", "A,B"]); + var allChecksPolicies = await GetRepositoryPolicies(repoUrl, _branchName); + var expectedAllChecksPolicies = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n AllChecksSuccessful\r\n ignoreChecks = \r\n" + + " [\r\n" + + " \"A\",\r\n" + + " \"B\"\r\n" + + " ]\r\n"; + allChecksPolicies.Should().BeEquivalentTo(expectedAllChecksPolicies, "Repository policy is incorrect for all checks successful case"); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs new file mode 100644 index 0000000000..4bdea79004 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_SdkUpdate.cs @@ -0,0 +1,136 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using FluentAssertions; +using Microsoft.DotNet.DarcLib; +using NUnit.Framework; +using ProductConstructionService.Client.Models; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[NonParallelizable] +internal class ScenarioTests_SdkUpdate : ScenarioTestBase +{ + private TestParameters _parameters; + private readonly Random _random = new(); + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [TestCase(false)] + [TestCase(true)] + public async Task ArcadeSdkUpdate_E2E(bool targetAzDO) + { + _parameters = await TestParameters.GetAsync(); + SetTestParameters(_parameters); + + var testChannelName = "Test Channel " + _random.Next(int.MaxValue); + const string sourceRepo = "arcade"; + const string sourceRepoUri = "https://github.com/dotnet/arcade"; + const string sourceBranch = "dependencyflow-tests"; + const string sourceCommit = "0b36b99e29b1751403e23cfad0a7dff585818051"; + const string newArcadeSdkVersion = "2.1.0"; + var sourceBuildNumber = _random.Next(int.MaxValue).ToString(); + + ImmutableList sourceAssets = ImmutableList.Create() + .Add(new AssetData(true) + { + Name = DependencyFileManager.ArcadeSdkPackageName, + Version = newArcadeSdkVersion, + }); + var targetRepo = "maestro-test2"; + var targetBranch = "test/" + _random.Next(int.MaxValue).ToString(); + + await using AsyncDisposableValue channel = + await CreateTestChannelAsync(testChannelName).ConfigureAwait(false); + await using AsyncDisposableValue sub = + await CreateSubscriptionAsync(testChannelName, sourceRepo, targetRepo, targetBranch, "none", targetIsAzDo: targetAzDO); + Build build = + await CreateBuildAsync(GetRepoUrl("dotnet", sourceRepo), sourceBranch, sourceCommit, sourceBuildNumber, sourceAssets); + + await using IAsyncDisposable _ = await AddBuildToChannelAsync(build.Id, testChannelName); + + using TemporaryDirectory repo = targetAzDO + ? await CloneAzDoRepositoryAsync(targetRepo) + : await CloneRepositoryAsync(targetRepo); + + using (ChangeDirectory(repo.Directory)) + { + await RunGitAsync("checkout", "-b", targetBranch).ConfigureAwait(false); + await RunDarcAsync("add-dependency", + "--name", DependencyFileManager.ArcadeSdkPackageName, + "--type", "toolset", + "--repo", sourceRepoUri); + await RunGitAsync("commit", "-am", "Add dependencies."); + await using IAsyncDisposable ___ = await PushGitBranchAsync("origin", targetBranch); + await TriggerSubscriptionAsync(sub.Value); + + var expectedTitle = $"[{targetBranch}] Update dependencies from dotnet/arcade"; + DependencyDetail expectedDependency = new() + { + Name = DependencyFileManager.ArcadeSdkPackageName, + Version = newArcadeSdkVersion, + RepoUri = sourceRepoUri, + Commit = sourceCommit, + Type = DependencyType.Toolset, + Pinned = false, + }; + + if (targetAzDO) + { + await CheckAzDoPullRequest( + expectedTitle, + targetRepo, + targetBranch, + [expectedDependency], + repo.Directory, + isCompleted: false, + isUpdated: false, + expectedFeeds: null, + notExpectedFeeds: null); + return; + } + + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + pr.Title.Should().BeEquivalentTo(expectedTitle); + + await CheckoutRemoteRefAsync(pr.MergeCommitSha); + + var dependencies = await RunDarcAsync("get-dependencies"); + var dependencyLines = dependencies.Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries); + dependencyLines.Should().BeEquivalentTo( + [ + $"Name: {DependencyFileManager.ArcadeSdkPackageName}", + $"Version: {newArcadeSdkVersion}", + $"Repo: {sourceRepoUri}", + $"Commit: {sourceCommit}", + "Type: Toolset", + "Pinned: False", + ]); + + using TemporaryDirectory arcadeRepo = await CloneRepositoryAsync("dotnet", sourceRepo); + using (ChangeDirectory(arcadeRepo.Directory)) + { + await CheckoutRemoteRefAsync(sourceCommit); + } + + var arcadeFiles = Directory.EnumerateFileSystemEntries(Path.Join(arcadeRepo.Directory, "eng", "common"), + "*", SearchOption.AllDirectories) + .Select(s => s.Substring(arcadeRepo.Directory.Length)) + .ToHashSet(); + var repoFiles = Directory.EnumerateFileSystemEntries(Path.Join(repo.Directory, "eng", "common"), "*", + SearchOption.AllDirectories) + .Select(s => s.Substring(repo.Directory.Length)) + .ToHashSet(); + + arcadeFiles.Should().BeEquivalentTo(repoFiles); + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs new file mode 100644 index 0000000000..a839cb07de --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs @@ -0,0 +1,273 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.Darc; +using NUnit.Framework; +using NUnit.Framework.Internal; +using ProductConstructionService.Client.Models; +using ProductConstructionService.ScenarioTests.ObjectHelpers; + +namespace ProductConstructionService.ScenarioTests; + +[TestFixture] +[Category("PostDeployment")] +[Category("AzDO")] +[Parallelizable] +internal class ScenarioTests_Subscriptions : ScenarioTestBase +{ + private TestParameters _parameters; + + [TearDown] + public Task DisposeAsync() + { + _parameters.Dispose(); + return Task.CompletedTask; + } + + [Test] + public async Task Subscriptions_EndToEnd() + { + TestContext.WriteLine("Subscription management tests..."); + var repo1Name = TestRepository.TestRepo1Name; + var repo2Name = TestRepository.TestRepo2Name; + var channel1Name = $"SubscriptionEndToEnd_TestChannel1_{Environment.MachineName}"; + var channel2Name = $"SubscriptionEndToEnd_TestChannel2_{Environment.MachineName}"; + + _parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); + SetTestParameters(_parameters); + + var repo1Uri = GetGitHubRepoUrl(repo1Name); + var repo2Uri = GetGitHubRepoUrl(repo2Name); + var repo1AzDoUri = GetAzDoRepoUrl(repo1Name); + var targetBranch = $"SubscriptionEndToEnd_TargetBranch_{Environment.MachineName}"; + + TestContext.WriteLine($"Creating channels {channel1Name} and {channel2Name}"); + await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(channel1Name).ConfigureAwait(false)) + { + await using (AsyncDisposableValue channel2 = await CreateTestChannelAsync(channel2Name).ConfigureAwait(false)) + { + + TestContext.WriteLine("Testing various command line parameters of add-subscription"); + await using AsyncDisposableValue subscription1Id = await CreateSubscriptionAsync( + channel1Name, repo1Name, repo2Name, targetBranch, "everyWeek", "maestro-auth-test"); + + var expectedSubscription1 = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo2Uri, + targetBranch, + channel1Name, + subscription1Id.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.EveryWeek, + false); + + var expectedSubscription1Info = UxHelpers.GetTextSubscriptionDescription(expectedSubscription1, null); + + await ValidateSubscriptionInfo(subscription1Id.Value, expectedSubscription1Info); + + await using AsyncDisposableValue subscription2Id = await CreateSubscriptionAsync( + channel1Name, repo1Name, repo1Name, targetBranch, "none", "maestro-auth-test", + ["--all-checks-passed", "--no-requested-changes", "--ignore-checks", "WIP,license/cla"], targetIsAzDo: true); + + var expectedSubscription2 = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo1AzDoUri, + targetBranch, + channel1Name, + subscription2Id.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.None, + false, + [MergePolicyConstants.AllCheckSuccessfulMergePolicyName, MergePolicyConstants.NoRequestedChangesMergePolicyName], + ["WIP", "license/cla"]); + + var expectedSubscription2Info = UxHelpers.GetTextSubscriptionDescription(expectedSubscription2, null); + + await ValidateSubscriptionInfo(subscription2Id.Value, expectedSubscription2Info); + + await using AsyncDisposableValue subscription3Id = await CreateSubscriptionAsync( + channel2Name, repo1Name, repo2Name, targetBranch, "none", "maestro-auth-test", + ["--all-checks-passed", "--no-requested-changes", "--ignore-checks", "WIP,license/cla"]); + + var expectedSubscription3 = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo2Uri, + targetBranch, + channel2Name, + subscription3Id.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.None, + false, + [MergePolicyConstants.AllCheckSuccessfulMergePolicyName, MergePolicyConstants.NoRequestedChangesMergePolicyName], + ["WIP", "license/cla"]); + + var expectedSubscription3Info = UxHelpers.GetTextSubscriptionDescription(expectedSubscription3, null); + + await ValidateSubscriptionInfo(subscription3Id.Value, expectedSubscription3Info); + + // Disable the first two subscriptions, but not the third. + TestContext.WriteLine("Disable the subscriptions for test channel 1"); + await SetSubscriptionStatusByChannel(false, channel1Name); + + // Disable one by id (classic usage) to make sure that works + TestContext.WriteLine("Disable the third subscription by id"); + await SetSubscriptionStatusById(false, subscription3Id.Value); + + // Re-enable + TestContext.WriteLine("Enable the third subscription by id"); + await SetSubscriptionStatusById(true, subscription3Id.Value); + + (await GetSubscriptionInfo(subscription3Id.Value)).Should().Contain("Enabled: True", $"Expected subscription {subscription3Id} to be enabled"); + + // Mass delete the subscriptions. Delete the first two but not the third. + TestContext.WriteLine("Delete the subscriptions for test channel 1"); + var message = await DeleteSubscriptionsForChannel(channel1Name); + + // Check that there are no subscriptions against channel1 now + TestContext.WriteLine("Verify that there are no subscriptions in test channel 1"); + Assert.ThrowsAsync(async () => await GetSubscriptions(channel1Name), "Subscriptions for channel 1 were not deleted."); + + // Validate the third subscription, which should still exist + TestContext.WriteLine("Verify that the third subscription still exists, then delete it"); + await ValidateSubscriptionInfo(subscription3Id.Value, expectedSubscription3Info); + var message2 = await DeleteSubscriptionById(subscription3Id.Value); + + // Attempt to create a batchable subscription with merge policies. + // Should fail, merge policies are set separately for batched subs + TestContext.WriteLine("Attempt to create a batchable subscription with merge policies"); + Assert.ThrowsAsync(async () => + await CreateSubscriptionAsync(channel1Name, repo1Name, repo2Name, targetBranch, "none", additionalOptions: ["--standard-automerge", "--batchable"]), + "Attempt to create a batchable subscription with merge policies"); + + // Create a batchable subscription + TestContext.WriteLine("Create a batchable subscription"); + await using AsyncDisposableValue batchSubscriptionId = await CreateSubscriptionAsync( + channel1Name, repo1Name, repo2Name, targetBranch, "everyWeek", "maestro-auth-test", additionalOptions: ["--batchable"]); + + var expectedBatchedSubscription = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo2Uri, + targetBranch, + channel1Name, + batchSubscriptionId.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.EveryWeek, + true); + + var expectedBatchedSubscriptionInfo = UxHelpers.GetTextSubscriptionDescription(expectedBatchedSubscription, null); + + await ValidateSubscriptionInfo(batchSubscriptionId.Value, expectedBatchedSubscriptionInfo); + await DeleteSubscriptionById(batchSubscriptionId.Value); + + TestContext.WriteLine("Testing YAML for darc add-subscription"); + + var yamlDefinition = $@" + Channel: {channel1Name} + Source Repository URL: {repo1Uri} + Target Repository URL: {repo2Uri} + Target Branch: {targetBranch} + Update Frequency: everyWeek + Batchable: False + Merge Policies: + - Name: Standard + Source Enabled: False + Excluded Assets: [] + "; + + await using AsyncDisposableValue yamlSubscriptionId = await CreateSubscriptionAsync(yamlDefinition); + + var expectedYamlSubscription = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo2Uri, + targetBranch, + channel1Name, + yamlSubscriptionId.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.EveryWeek, + false, + [MergePolicyConstants.StandardMergePolicyName]); + + var expectedYamlSubscriptionInfo = UxHelpers.GetTextSubscriptionDescription(expectedYamlSubscription, null); + + await ValidateSubscriptionInfo(yamlSubscriptionId.Value, expectedYamlSubscriptionInfo); + await DeleteSubscriptionById(yamlSubscriptionId.Value); + + TestContext.WriteLine("Change casing of the various properties. Expecting no changes."); + + var yamlDefinition2 = $@" + Channel: {channel1Name} + Source Repository URL: {repo1Uri} + Target Repository URL: {repo2Uri} + Target Branch: {targetBranch} + Update Frequency: everyweek + Batchable: False + Merge Policies: + - Name: standard + Source Enabled: False + Excluded Assets: [] + "; + + await using AsyncDisposableValue yamlSubscription2Id = await CreateSubscriptionAsync(yamlDefinition2); + + var expectedYamlSubscription2 = SubscriptionBuilder.BuildSubscription( + repo1Uri, + repo2Uri, + targetBranch, + channel1Name, + yamlSubscription2Id.Value, + Microsoft.DotNet.Maestro.Client.Models.UpdateFrequency.EveryWeek, false, + [MergePolicyConstants.StandardMergePolicyName]); + + var expectedYamlSubscriptionInfo2 = UxHelpers.GetTextSubscriptionDescription(expectedYamlSubscription2, null); + + await ValidateSubscriptionInfo(yamlSubscription2Id.Value, expectedYamlSubscriptionInfo2); + await DeleteSubscriptionById(yamlSubscription2Id.Value); + + TestContext.WriteLine("Attempt to add multiple of the same merge policy checks. Should fail."); + + var yamlDefinition3 = $""" + Channel: {channel1Name} + Source Repository URL: {repo1Uri} + Target Repository URL: {repo2Uri} + Target Branch: {targetBranch} + Update Frequency: everyweek + Batchable: False + Merge Policies: + - Name: AllChecksSuccessful + Properties: + ignoreChecks: + - WIP + - license/cla + - Name: AllChecksSuccessful + Properties: + ignoreChecks: + - WIP + - MySpecialCheck + Source Enabled: False + Excluded Assets: [] + """; + + Assert.ThrowsAsync(async () => + await CreateSubscriptionAsync(yamlDefinition3), "Attempt to create a subscription with multiples of the same merge policy."); + + TestContext.WriteLine("Testing duplicate subscription handling..."); + AsyncDisposableValue yamlSubscription3Id = await CreateSubscriptionAsync(channel1Name, repo1Name, repo2Name, targetBranch, "everyWeek", "maestro-auth-test"); + + Assert.ThrowsAsync(async () => + await CreateSubscriptionAsync(channel1Name, repo1Name, repo2Name, targetBranch, "everyWeek", "maestro-auth-test"), + "Attempt to create a subscription with the same values as an existing subscription."); + + Assert.ThrowsAsync(async () => + await CreateSubscriptionAsync(channel1Name, repo1Name, repo2Name, targetBranch, "everyweek", "maestro-auth-test"), + "Attempt to create a subscription with the same values as an existing subscription (except for the casing of one parameter."); + + await DeleteSubscriptionById(yamlSubscription3Id.Value); + + TestContext.WriteLine("End of test case. Starting clean up."); + } + } + } + + private async Task ValidateSubscriptionInfo(string subscriptionId, string expectedSubscriptionInfo) + { + var subscriptionInfo = await GetSubscriptionInfo(subscriptionId); + subscriptionInfo.Should().BeEquivalentTo(expectedSubscriptionInfo); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/Shareable.cs b/test/ProductConstructionService.ScenarioTests/Shareable.cs new file mode 100644 index 0000000000..728ff5b815 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/Shareable.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +public sealed class Shareable : IDisposable where T : class, IDisposable +{ + private T _inner; + + internal Shareable(T inner) + { + _inner = inner; + } + + public void Dispose() + { + _inner?.Dispose(); + } + + public T TryTake() + { + return Interlocked.Exchange(ref _inner, null); + } + + public T Peek() + { + if (_inner == null) + { + throw new InvalidOperationException("Peek() called on null Shareable."); + } + return _inner; + } +} + +public static class Shareable +{ + public static Shareable Create(T target) where T : class, IDisposable + { + return new Shareable(target); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/TemporaryDirectory.cs b/test/ProductConstructionService.ScenarioTests/TemporaryDirectory.cs new file mode 100644 index 0000000000..daf2da1f9a --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/TemporaryDirectory.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +public class TemporaryDirectory : IDisposable +{ + public static TemporaryDirectory Get() + { + var dir = Path.Join(Path.GetTempPath(), Path.GetRandomFileName()); + System.IO.Directory.CreateDirectory(dir); + return new TemporaryDirectory(dir); + } + + public string Directory { get; } + + private TemporaryDirectory(string dir) + { + Directory = dir; + } + + public void Dispose() + { + try + { + System.IO.Directory.Delete(Directory, true); + } + catch (UnauthorizedAccessException) + { + Thread.Sleep(5000); + try + { + System.IO.Directory.Delete(Directory, true); + } + catch (UnauthorizedAccessException) + { + // We tried, don't fail the test. + } + } + } +} diff --git a/test/ProductConstructionService.ScenarioTests/TestHelpers.cs b/test/ProductConstructionService.ScenarioTests/TestHelpers.cs new file mode 100644 index 0000000000..0fee1e363c --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/TestHelpers.cs @@ -0,0 +1,162 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text; +using NUnit.Framework; + +namespace ProductConstructionService.ScenarioTests; + +public static class TestHelpers +{ + public static async Task RunExecutableAsync(string executable, params string[] args) + { + return await RunExecutableAsyncWithInput(executable, "", args); + } + + public static async Task RunExecutableAsyncWithInput(string executable, string input, params string[] args) + { + TestContext.WriteLine(FormatExecutableCall(executable, args)); + var output = new StringBuilder(); + + void WriteOutput(string message) + { + if (message != null) + { + Debug.WriteLine(message); + output.AppendLine(message); + TestContext.WriteLine(message); + } + } + + var psi = new ProcessStartInfo(executable) + { + RedirectStandardError = true, + RedirectStandardOutput = true, + RedirectStandardInput = true, + UseShellExecute = false + }; + + foreach (var arg in args) + { + // The string append used by Process will accept null params + // and then throw without identifying the param, so we need to handle it here to get logging + if (arg != null) + { + psi.ArgumentList.Add(arg); + } + else + { + WriteOutput("Null parameter encountered while constructing command string."); + } + } + + using var process = new Process + { + StartInfo = psi + }; + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + process.EnableRaisingEvents = true; + process.Exited += (s, e) => { tcs.TrySetResult(true); }; + process.Start(); + + Task exitTask = tcs.Task; + var stdin = Task.Run(() => { process.StandardInput.Write(input); process.StandardInput.Close(); }); + Task stdout = process.StandardOutput.ReadLineAsync(); + Task stderr = process.StandardError.ReadLineAsync(); + var list = new List { exitTask, stdout, stderr, stdin }; + while (list.Count != 0) + { + var done = await Task.WhenAny(list); + list.Remove(done); + if (done == exitTask) + { + continue; + } + + if (done == stdout) + { + var data = await stdout; + WriteOutput(data); + if (data != null) + { + list.Add(stdout = process.StandardOutput.ReadLineAsync()); + } + continue; + } + + if (done == stderr) + { + var data = await stderr; + WriteOutput(data); + if (data != null) + { + list.Add(stderr = process.StandardError.ReadLineAsync()); + } + continue; + } + + if (done == stdin) + { + await stdin; + continue; + } + + throw new InvalidOperationException("Unexpected Task completed."); + } + + if (process.ExitCode != 0) + { + var exceptionWithConsoleLog = new ScenarioTestException($"{executable} exited with code {process.ExitCode}"); + exceptionWithConsoleLog.Data.Add("ConsoleOutput", output.ToString()); + throw exceptionWithConsoleLog; + } + + return output.ToString(); + } + + public static async Task Which(string command) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + var cmd = Environment.GetEnvironmentVariable("ComSpec") ?? "cmd"; + return (await RunExecutableAsync(cmd, "/c", $"where {command}")).Trim() + // get the first line of where's output + .Split(new[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() ?? ""; + } + + return (await RunExecutableAsync("/bin/sh", "-c", $"which {command}")).Trim(); + } + + internal static string FormatExecutableCall(string executable, params string[] args) + { + var output = new StringBuilder(); + var secretArgNames = new[] { "-p", "--password", "--github-pat", "--azdev-pat" }; + + output.Append(executable); + for (var i = 0; i < args.Length; i++) + { + output.Append(' '); + + if (i > 0 && secretArgNames.Contains(args[i - 1])) + { + output.Append("\"***\""); + continue; + } + + output.Append($"\"{args[i]}\""); + } + + return output.ToString(); + } +} + +public class ScenarioTestException : Exception +{ + public ScenarioTestException(string message) + { + TestContext.WriteLine(message); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/TestHelpersTest.cs b/test/ProductConstructionService.ScenarioTests/TestHelpersTest.cs new file mode 100644 index 0000000000..85ac2e8551 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/TestHelpersTest.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using NUnit.Framework; + +namespace ProductConstructionService.ScenarioTests; + +public class TestHelpersTests +{ + [Test] + public void EmptyArguments() + { + var formatted = TestHelpers.FormatExecutableCall("darc.exe"); + + formatted.Should().Be("darc.exe"); + } + + [Test] + public void HarmlessArguments() + { + var formatted = TestHelpers.FormatExecutableCall("darc.exe", ["add-channel", "--name", "what-a-channel"]); + + formatted.Should().Be("darc.exe \"add-channel\" \"--name\" \"what-a-channel\""); + } + + [Test] + public void ArgumentsWithSecretTokensInside() + { + var formatted = TestHelpers.FormatExecutableCall("darc.exe", ["-p", "secret", "add-channel", "--github-pat", "another secret", "--name", "what-a-channel"]); + + formatted.Should().Be("darc.exe \"-p\" \"***\" \"add-channel\" \"--github-pat\" \"***\" \"--name\" \"what-a-channel\""); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/TestParameters.cs b/test/ProductConstructionService.ScenarioTests/TestParameters.cs new file mode 100644 index 0000000000..cb18c330ec --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/TestParameters.cs @@ -0,0 +1,179 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; +using System.Runtime.InteropServices; +using Maestro.Common.AzureDevOpsTokens; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.Internal.Testing.Utility; +using Microsoft.Extensions.Configuration; +using Octokit.Internal; +using ProductConstructionService.Client; + +#nullable enable +namespace ProductConstructionService.ScenarioTests; + +public class TestParameters : IDisposable +{ + internal readonly TemporaryDirectory _dir; + + private static readonly string pcsBaseUri; + private static readonly string? pcsToken; + private static readonly string githubToken; + private static readonly string darcPackageSource; + private static readonly string azdoToken; + private static readonly bool isCI; + private static readonly string? darcDir; + private static readonly string? darcVersion; + + static TestParameters() + { + IConfiguration userSecrets = new ConfigurationBuilder() + .AddUserSecrets() + .Build(); + + pcsBaseUri = Environment.GetEnvironmentVariable("PCS_BASEURI") + ?? userSecrets["PCS_BASEURI"] + ?? "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; + pcsToken = Environment.GetEnvironmentVariable("PCS_TOKEN") ?? userSecrets["PCS_TOKEN"]; + isCI = Environment.GetEnvironmentVariable("DARC_IS_CI")?.ToLower() == "true"; + githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN") ?? userSecrets["GITHUB_TOKEN"] + ?? throw new Exception("Please configure the GitHub token"); + darcPackageSource = Environment.GetEnvironmentVariable("DARC_PACKAGE_SOURCE") ?? userSecrets["DARC_PACKAGE_SOURCE"] + ?? throw new Exception("Please configure the Darc package source"); + azdoToken = Environment.GetEnvironmentVariable("AZDO_TOKEN") ?? userSecrets["AZDO_TOKEN"] + ?? throw new Exception("Please configure the Azure DevOps token"); + darcDir = Environment.GetEnvironmentVariable("DARC_DIR"); + darcVersion = Environment.GetEnvironmentVariable("DARC_VERSION") ?? userSecrets["DARC_VERSION"]; + + } + + /// If set to true, the test will attempt to use the non primary endpoint, if provided + public static async Task GetAsync(bool useNonPrimaryEndpoint = false) + { + var testDir = TemporaryDirectory.Get(); + var testDirSharedWrapper = Shareable.Create(testDir); + + IProductConstructionServiceApi pcsApi = pcsToken != null + ? PcsApiFactory.GetAuthenticated(pcsBaseUri, pcsToken, managedIdentityId: null, disableInteractiveAuth: isCI) + : PcsApiFactory.GetAnonymous(pcsBaseUri); + + var darcRootDir = darcDir; + if (string.IsNullOrEmpty(darcRootDir)) + { + await InstallDarc(pcsApi, testDirSharedWrapper); + darcRootDir = testDirSharedWrapper.Peek()!.Directory; + } + + var darcExe = Path.Join(darcRootDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc"); + var git = await TestHelpers.Which("git"); + var tokenProvider = AzureDevOpsTokenProvider.FromStaticOptions(new() + { + ["default"] = new() + { + Token = azdoToken + } + }); + + Assembly assembly = typeof(TestParameters).Assembly; + var githubApi = + new Octokit.GitHubClient( + new Octokit.ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute()?.InformationalVersion), + new InMemoryCredentialStore(new Octokit.Credentials(githubToken))); + var azDoClient = + new AzureDevOpsClient( + tokenProvider, + new ProcessManager(new NUnitLogger(), git), + new NUnitLogger(), + testDirSharedWrapper.TryTake()!.Directory); + + return new TestParameters( + darcExe, git, pcsBaseUri, pcsToken, githubToken, pcsApi, githubApi, azDoClient, testDir, azdoToken, isCI); + } + + private static async Task InstallDarc(IProductConstructionServiceApi pcsApi, Shareable toolPath) + { + var darcVersion = TestParameters.darcVersion ?? await pcsApi.Assets.GetDarcVersionAsync(); + var dotnetExe = await TestHelpers.Which("dotnet"); + + var toolInstallArgs = new List + { + "tool", "install", + "--version", darcVersion, + "--tool-path", toolPath.Peek()!.Directory, + "Microsoft.DotNet.Darc", + }; + + if (!string.IsNullOrEmpty(darcPackageSource)) + { + toolInstallArgs.Add("--add-source"); + toolInstallArgs.Add(darcPackageSource); + } + + await TestHelpers.RunExecutableAsync(dotnetExe, [.. toolInstallArgs]); + } + + private TestParameters( + string darcExePath, + string gitExePath, + string pcsBaseUri, + string? pcsToken, + string gitHubToken, + IProductConstructionServiceApi pcsApi, + Octokit.GitHubClient gitHubApi, + AzureDevOpsClient azdoClient, + TemporaryDirectory dir, + string azdoToken, + bool isCI) + { + _dir = dir; + DarcExePath = darcExePath; + GitExePath = gitExePath; + MaestroBaseUri = pcsBaseUri; + MaestroToken = pcsToken; + GitHubToken = gitHubToken; + PcsApi = pcsApi; + GitHubApi = gitHubApi; + AzDoClient = azdoClient; + AzDoToken = azdoToken; + IsCI = isCI; + } + + public string DarcExePath { get; } + + public string GitExePath { get; } + + public string GitHubUser { get; } = "dotnet-maestro-bot"; + + public string GitHubTestOrg { get; } = "maestro-auth-test"; + + public string MaestroBaseUri { get; } + + public string? MaestroToken { get; } + + public string GitHubToken { get; } + + public IProductConstructionServiceApi PcsApi { get; } + + public Octokit.GitHubClient GitHubApi { get; } + + public AzureDevOpsClient AzDoClient { get; } + + public int AzureDevOpsBuildDefinitionId { get; } = 6; + + public int AzureDevOpsBuildId { get; } = 144618; + + public string AzureDevOpsAccount { get; } = "dnceng"; + + public string AzureDevOpsProject { get; } = "internal"; + + public string AzDoToken { get; } + + public bool IsCI { get; } + + public void Dispose() + { + _dir?.Dispose(); + } +} diff --git a/test/ProductConstructionService.ScenarioTests/TestRepository.cs b/test/ProductConstructionService.ScenarioTests/TestRepository.cs new file mode 100644 index 0000000000..7b49f4f2d5 --- /dev/null +++ b/test/ProductConstructionService.ScenarioTests/TestRepository.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.ScenarioTests; + +internal class TestRepository +{ + internal static string TestRepo1Name => "maestro-test1"; + internal static string TestRepo2Name => "maestro-test2"; + internal static string TestRepo3Name => "maestro-test3"; + internal static string SourceBranch => "master"; + + // This branch and commit data is special for the coherency test + // It's required to make sure that the dependency tree is set up correctly in the repo without conflicting with other test cases + internal static string CoherencySourceBranch => "coherency-tree"; + internal static string CoherencyTestRepo1Commit => "cc1a27107a1f4c4bc5e2f796c5ef346f60abb404"; + internal static string CoherencyTestRepo2Commit => "8460158878d4b7568f55d27960d4453877523ea6"; +} From fc22fa3af82f0fd52e5670a335e4c27b339e37ae Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:21:04 +0200 Subject: [PATCH 14/33] Port feed cleaner to PCS (#3892) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- arcade-services.sln | 30 ++ ...pipelines-product-construction-service.yml | 2 +- .../provision.bicep | 62 +++- .../scheduledContainerJob.bicep | 2 +- .../FeedCleanerService/FeedCleanerService.cs | 11 +- .../ProductConstructionService.Api/Dockerfile | 12 +- .../AddJobLoggingExtension.cs | 16 +- .../FakeAzureDevOpsClient.cs | 42 +++ .../FeedCleaner.cs | 310 +++++++++++++++++ .../FeedCleanerConfiguration.cs | 42 +++ .../FeedCleanerOptions.cs | 15 + ...ductConstructionService.FeedCleaner.csproj | 28 ++ .../Program.cs | 26 ++ .../appsettings.Development.json | 8 + .../appsettings.Staging.json | 9 + .../appsettings.json | 27 ++ .../LongestBuildPathUpdaterConfiguration.cs | 2 +- .../Program.cs | 1 - .../SubscriptionTriggererConfiguration.cs | 2 +- .../DependencyRegistrationTest.cs | 37 +++ .../FeedCleanerTests.cs | 311 ++++++++++++++++++ ...nstructionService.FeedCleaner.Tests.csproj | 21 ++ 22 files changed, 991 insertions(+), 25 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/FakeAzureDevOpsClient.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/Program.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Development.json create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Staging.json create mode 100644 src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json create mode 100644 test/ProductConstructionService.FeedCleaner.Tests/DependencyRegistrationTest.cs create mode 100644 test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs create mode 100644 test/ProductConstructionService.FeedCleaner.Tests/ProductConstructionService.FeedCleaner.Tests.csproj diff --git a/arcade-services.sln b/arcade-services.sln index 39b3d91126..2d491b556e 100644 --- a/arcade-services.sln +++ b/arcade-services.sln @@ -138,6 +138,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.DependencyFlow.Tests", "test\ProductConstructionService.DependencyFlow.Tests\ProductConstructionService.DependencyFlow.Tests.csproj", "{045BBB97-5A75-443A-8447-2035CBC0549A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.FeedCleaner", "src\ProductConstructionService\ProductConstructionService.FeedCleaner\ProductConstructionService.FeedCleaner.csproj", "{B5833185-DD07-4C64-A4DA-D8290294F7E7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.FeedCleaner.Tests", "test\ProductConstructionService.FeedCleaner.Tests\ProductConstructionService.FeedCleaner.Tests.csproj", "{D94319F8-FCA0-495B-8B6E-190B4EEBEF93}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.ScenarioTests", "test\ProductConstructionService.ScenarioTests\ProductConstructionService.ScenarioTests.csproj", "{12D91D30-EC50-4D2B-8D67-0C19FCD2303F}" EndProject Global @@ -667,6 +671,30 @@ Global {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x64.Build.0 = Release|Any CPU {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.ActiveCfg = Release|Any CPU {045BBB97-5A75-443A-8447-2035CBC0549A}.Release|x86.Build.0 = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|x64.ActiveCfg = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|x64.Build.0 = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|x86.ActiveCfg = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Debug|x86.Build.0 = Debug|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|Any CPU.Build.0 = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|x64.ActiveCfg = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|x64.Build.0 = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|x86.ActiveCfg = Release|Any CPU + {B5833185-DD07-4C64-A4DA-D8290294F7E7}.Release|x86.Build.0 = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|x64.ActiveCfg = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|x64.Build.0 = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|x86.ActiveCfg = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Debug|x86.Build.0 = Debug|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|Any CPU.Build.0 = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|x64.ActiveCfg = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|x64.Build.0 = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|x86.ActiveCfg = Release|Any CPU + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93}.Release|x86.Build.0 = Release|Any CPU {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|Any CPU.Build.0 = Debug|Any CPU {12D91D30-EC50-4D2B-8D67-0C19FCD2303F}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -731,6 +759,8 @@ Global {29A75658-2DC4-4E85-8A53-97198F00F28D} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} {E312686C-A134-486F-9F62-89CE6CA34702} = {243A4561-BF35-405A-AF12-AC57BB27796D} {045BBB97-5A75-443A-8447-2035CBC0549A} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} + {B5833185-DD07-4C64-A4DA-D8290294F7E7} = {243A4561-BF35-405A-AF12-AC57BB27796D} + {D94319F8-FCA0-495B-8B6E-190B4EEBEF93} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} {12D91D30-EC50-4D2B-8D67-0C19FCD2303F} = {1A456CF0-C09A-4DE6-89CE-1110EED31180} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index 02d64b5e50..c98f6d24e3 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -26,7 +26,7 @@ variables: - name: containerappName value: product-construction-int - name: containerjobNames - value: sub-triggerer-twicedaily-int,sub-triggerer-daily-int,sub-triggerer-weekly-int,longest-path-updater-job-int + value: sub-triggerer-twicedaily-int,sub-triggerer-daily-int,sub-triggerer-weekly-int,longest-path-updater-job-int,feed-cleaner-int - name: containerRegistryName value: productconstructionint - name: containerappEnvironmentName diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 1f3e4b8199..029a56963b 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -52,7 +52,7 @@ param pcsIdentityName string = 'ProductConstructionServiceInt' param deploymentIdentityName string = 'ProductConstructionServiceDeploymentInt' @description('Bicep requires an image when creating a containerapp. Using a dummy image for that.') -var containerImageName = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' +param containerImageName = 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' @description('Virtual network name') param virtualNetworkName string = 'product-construction-service-vnet-int' @@ -78,6 +78,12 @@ param longestBuildPathUpdaterIdentityName string = 'LongestBuildPathUpdaterInt' @description('Longest Build Path Updater Job Name') param longestBuildPathUpdaterJobName string = 'longest-path-updater-job-int' +@description('Feed Cleaner Job name') +param feedCleanerJobName string = 'feed-cleaner-int' + +@description('Feed Cleaner Identity name') +param feedCleanerIdentityName string = 'FeedCleanerInt' + @description('Network security group name') param networkSecurityGroupName string = 'product-construction-service-nsg-int' @@ -357,23 +363,28 @@ resource containerRegistry 'Microsoft.ContainerRegistry/registries@2022-02-01-pr resource deploymentIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: deploymentIdentityName - location: location + name: deploymentIdentityName + location: location } resource pcsIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: pcsIdentityName - location: location + name: pcsIdentityName + location: location } resource subscriptionTriggererIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: subscriptionTriggererIdentityName - location: location + name: subscriptionTriggererIdentityName + location: location } resource longestBuildPathUpdaterIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: longestBuildPathUpdaterIdentityName - location: location + name: longestBuildPathUpdaterIdentityName + location: location +} + +resource feedCleanerIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: feedCleanerIdentityName + location: location } // allow acr pulls to the identity used for the aca's @@ -409,6 +420,17 @@ resource longestBuildPathUpdaterIdentityAcrPull 'Microsoft.Authorization/roleAss } } +// allow acr pulls to the identity used for the feed cleaner +resource feedCleanerIdentityAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + scope: containerRegistry // Use when specifying a scope that is different than the deployment scope + name: guid(subscription().id, resourceGroup().id, acrPullRole) + properties: { + roleDefinitionId: acrPullRole + principalType: 'ServicePrincipal' + principalId: feedCleanerIdentity.properties.principalId + } +} + // azure system role for setting up acr pull access var acrPullRole = subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') @@ -655,6 +677,28 @@ module longestBuildPathUpdater 'scheduledContainerJob.bicep' = { ] } +module feedCleaner 'scheduledContainerJob.bicep' = { + name: 'feedCleaner' + params: { + jobName: feedCleanerJobName + location: location + aspnetcoreEnvironment: aspnetcoreEnvironment + applicationInsightsConnectionString: applicationInsights.properties.ConnectionString + userAssignedIdentityId: feedCleanerIdentity.id + cronSchedule: '0 2 * * *' + containerRegistryName: containerRegistryName + containerAppsEnvironmentId: containerAppsEnvironment.id + containerImageName: containerImageName + dllFullPath: '/app/FeedCleaner/ProductConstructionService.FeedCleaner.dll' + contributorRoleId: contributorRole + deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId + } + dependsOn: [ + feedCleanerIdentityAcrPull + longestBuildPathUpdater + ] +} + resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { name: keyVaultName location: location diff --git a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep index 8815949f85..0244ed32ec 100644 --- a/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep +++ b/eng/service-templates/ProductConstructionService/scheduledContainerJob.bicep @@ -8,7 +8,7 @@ param containerRegistryName string param containerAppsEnvironmentId string param containerImageName string param dllFullPath string -param argument string +param argument string = '' param contributorRoleId string param deploymentIdentityPrincipalId string diff --git a/src/Maestro/FeedCleanerService/FeedCleanerService.cs b/src/Maestro/FeedCleanerService/FeedCleanerService.cs index 3f540f3002..0b51be8dd7 100644 --- a/src/Maestro/FeedCleanerService/FeedCleanerService.cs +++ b/src/Maestro/FeedCleanerService/FeedCleanerService.cs @@ -68,7 +68,16 @@ public async Task CleanManagedFeedsAsync() foreach (var azdoAccount in Options.AzdoAccounts) { - List allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount); + List allFeeds; + try + { + allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount); + } + catch (Exception ex) + { + Logger.LogError(ex, $"Failed to get feeds for account {azdoAccount}"); + continue; + } IEnumerable managedFeeds = allFeeds.Where(f => Regex.IsMatch(f.Name, FeedConstants.MaestroManagedFeedNamePattern)); foreach (var feed in managedFeeds) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile index 0f51b8d432..1e138d62a8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile +++ b/src/ProductConstructionService/ProductConstructionService.Api/Dockerfile @@ -27,14 +27,16 @@ WORKDIR /src/ProductConstructionService COPY ["src/ProductConstructionService/ProductConstructionService.Api/ProductConstructionService.Api.csproj", "./ProductConstructionService.Api/"] COPY ["src/ProductConstructionService/ProductConstructionService.Common/ProductConstructionService.Common.csproj", "./ProductConstructionService.Common/"] COPY ["src/ProductConstructionService/ProductConstructionService.DependencyFlow/ProductConstructionService.DependencyFlow.csproj", "./ProductConstructionService.DependencyFlow/"] +COPY ["src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj", "./ProductConstructionService.FeedCleaner/"] COPY ["src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj", "./ProductConstructionService.LongestBuildPathUpdater/"] COPY ["src/ProductConstructionService/ProductConstructionService.ServiceDefaults/ProductConstructionService.ServiceDefaults.csproj", "./ProductConstructionService.ServiceDefaults/"] COPY ["src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj", "./ProductConstructionService.SubscriptionTriggerer/"] COPY ["src/ProductConstructionService/ProductConstructionService.WorkItems/ProductConstructionService.WorkItems.csproj", "./ProductConstructionService.WorkItems/"] RUN dotnet restore "./ProductConstructionService.Api/ProductConstructionService.Api.csproj" -RUN dotnet restore "./ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj" +RUN dotnet restore "./ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj" RUN dotnet restore "./ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj" +RUN dotnet restore "./ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj" WORKDIR /src/Maestro COPY ["src/Maestro/Client/src", "./Client/src"] @@ -57,14 +59,18 @@ WORKDIR ./ProductConstructionService.Api RUN dotnet build -c $BUILD_CONFIGURATION -o /app/build --no-restore RUN dotnet publish -c $BUILD_CONFIGURATION -o /app/publish/ProductConstructionService /p:UseAppHost=false -WORKDIR ../ProductConstructionService.SubscriptionTriggerer +WORKDIR ../ProductConstructionService.FeedCleaner RUN dotnet build -c $BUILD_CONFIGURATION -o /app/build --no-restore -RUN dotnet publish -c $BUILD_CONFIGURATION -o /app/publish/SubscriptionTriggerer /p:UseAppHost=false +RUN dotnet publish -c $BUILD_CONFIGURATION -o /app/publish/FeedCleaner /p:UseAppHost=false WORKDIR ../ProductConstructionService.LongestBuildPathUpdater RUN dotnet build -c $BUILD_CONFIGURATION -o /app/build --no-restore RUN dotnet publish -c $BUILD_CONFIGURATION -o /app/publish/LongestBuildPathUpdater /p:UseAppHost=false +WORKDIR ../ProductConstructionService.SubscriptionTriggerer +RUN dotnet build -c $BUILD_CONFIGURATION -o /app/build --no-restore +RUN dotnet publish -c $BUILD_CONFIGURATION -o /app/publish/SubscriptionTriggerer /p:UseAppHost=false + # Build Angular app FROM mcr.microsoft.com/devcontainers/typescript-node:dev-20 AS angular WORKDIR /maestro-angular diff --git a/src/ProductConstructionService/ProductConstructionService.Common/AddJobLoggingExtension.cs b/src/ProductConstructionService/ProductConstructionService.Common/AddJobLoggingExtension.cs index c93f12d799..25fccc958a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/AddJobLoggingExtension.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/AddJobLoggingExtension.cs @@ -5,6 +5,7 @@ using Microsoft.ApplicationInsights.Channel; using Microsoft.ApplicationInsights.Extensibility; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace ProductConstructionService.Common; @@ -17,14 +18,14 @@ public static class AddJobLoggingExtension /// /// Telemetry channel where logs will be buffered. Needs to be flushed at the end of the app /// - public static IServiceCollection RegisterLogging( - this IServiceCollection services, - ITelemetryChannel telemetryChannel, - bool isDevelopment) + public static IHostApplicationBuilder RegisterLogging( + this IHostApplicationBuilder builder, + ITelemetryChannel telemetryChannel) { + bool isDevelopment = builder.Environment.IsDevelopment(); if (!isDevelopment) { - services.Configure( + builder.Services.Configure( config => { config.ConnectionString = Environment.GetEnvironmentVariable(ApplicationInsightsConnectionString); @@ -33,7 +34,7 @@ public static IServiceCollection RegisterLogging( ); } - services.AddLogging(builder => + builder.Services.AddLogging(builder => { if (!isDevelopment) { @@ -42,6 +43,7 @@ public static IServiceCollection RegisterLogging( // Console logging will be useful if we're investigating Console logs of a single job run builder.AddConsole(); }); - return services; + + return builder; } } diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FakeAzureDevOpsClient.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FakeAzureDevOpsClient.cs new file mode 100644 index 0000000000..315ff7c34a --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FakeAzureDevOpsClient.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; +using Newtonsoft.Json.Linq; + +namespace ProductConstructionService.FeedCleaner; + +// TODO (https://github.com/dotnet/arcade-services/issues/3808) delete this class and use the normal AzureDevOpsClient +internal class FakeAzureDevOpsClient : IAzureDevOpsClient +{ + public Task AdjustReleasePipelineArtifactSourceAsync(string accountName, string projectName, AzureDevOpsReleaseDefinition releaseDefinition, AzureDevOpsBuild build) + => Task.FromResult(releaseDefinition); + public Task DeleteFeedAsync(string accountName, string project, string feedIdentifier) => Task.CompletedTask; + public Task DeleteNuGetPackageVersionFromFeedAsync(string accountName, string project, string feedIdentifier, string packageName, string version) + => Task.CompletedTask; + public Task> GetBuildArtifactsAsync(string accountName, string projectName, int buildId, int maxRetries = 15) + => Task.FromResult>([]); + public Task GetBuildAsync(string accountName, string projectName, long buildId) + => Task.FromResult(new AzureDevOpsBuild()); + public Task GetBuildsAsync(string account, string project, int definitionId, string branch, int count, string status) + => Task.FromResult(new JObject()); + public Task GetFeedAndPackagesAsync(string accountName, string project, string feedIdentifier) + => Task.FromResult(new AzureDevOpsFeed("fake", "fake", "fake")); + public Task GetFeedAsync(string accountName, string project, string feedIdentifier) + => Task.FromResult(new AzureDevOpsFeed("fake", "fake", "fake")); + public Task> GetFeedsAndPackagesAsync(string accountName) + => Task.FromResult>([]); + public Task> GetFeedsAsync(string accountName) + => Task.FromResult>([]); + public Task> GetPackagesForFeedAsync(string accountName, string project, string feedIdentifier) + => Task.FromResult>([]); + public Task GetProjectIdAsync(string accountName, string projectName) => Task.FromResult(string.Empty); + public Task GetReleaseAsync(string accountName, string projectName, int releaseId) + => Task.FromResult(new AzureDevOpsRelease()); + public Task GetReleaseDefinitionAsync(string accountName, string projectName, long releaseDefinitionId) + => Task.FromResult(new AzureDevOpsReleaseDefinition()); + public Task StartNewBuildAsync(string accountName, string projectName, int buildDefinitionId, string sourceBranch, string sourceVersion, Dictionary queueTimeVariables, Dictionary templateParameters, Dictionary pipelineResources) + => Task.FromResult(0); + public Task StartNewReleaseAsync(string accountName, string projectName, AzureDevOpsReleaseDefinition releaseDefinition, int barBuildId) + => Task.FromResult(0); +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs new file mode 100644 index 0000000000..731b999ada --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleaner.cs @@ -0,0 +1,310 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net; +using System.Text.RegularExpressions; +using Maestro.Data; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace ProductConstructionService.FeedCleaner; + +public class FeedCleaner +{ + private BuildAssetRegistryContext _context; + private readonly HttpClient _httpClient; + private readonly IAzureDevOpsClient _azureDevOpsClient; + private readonly IOptions _options; + private ILogger _logger; + + public FeedCleaner( + BuildAssetRegistryContext context, + IAzureDevOpsClient azureDevOpsClient, + IOptions options, + ILogger logger) + { + _context = context; + _azureDevOpsClient = azureDevOpsClient; + _options = options; + _logger = logger; + _httpClient = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }); + } + + private FeedCleanerOptions Options => _options.Value; + + public async Task CleanManagedFeedsAsync() + { + if (!Options.Enabled) + { + _logger.LogInformation("Feed cleaner service is disabled in this environment"); + return; + } + + Dictionary>> packagesInReleaseFeeds = + await GetPackagesForReleaseFeedsAsync(); + + foreach (var azdoAccount in Options.AzdoAccounts) + { + List allFeeds; + try + { + allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Failed to get feeds for account {azdoAccount}"); + continue; + } + IEnumerable managedFeeds = allFeeds.Where(f => Regex.IsMatch(f.Name, FeedConstants.MaestroManagedFeedNamePattern)); + + foreach (var feed in managedFeeds) + { + try + { + await PopulatePackagesForFeedAsync(feed); + foreach (var package in feed.Packages) + { + HashSet updatedVersions = + await UpdateReleasedVersionsForPackageAsync(feed, package, packagesInReleaseFeeds); + + await DeletePackageVersionsFromFeedAsync(feed, package.Name, updatedVersions); + } + // We may have deleted all packages in the previous operation, if so, we should delete the feed, + // refresh the packages in the feed to check this. + await PopulatePackagesForFeedAsync(feed); + } + catch (Exception ex) + { + _logger.LogError(ex, $"Something failed while trying to update the released packages in feed {feed.Name}"); + } + } + } + } + + /// + /// Get a mapping of feed -> (package, versions) for the release feeds so it + /// can be easily queried whether a version of a package is in a feed. + /// + /// Mapping of packages to versions for the release feeds. + private async Task>>> GetPackagesForReleaseFeedsAsync() + { + var packagesWithVersionsInReleaseFeeds = new Dictionary>>(); + IEnumerable dotnetManagedFeeds = Options.ReleasePackageFeeds; + foreach ((string account, string project, string feedName) in dotnetManagedFeeds) + { + string readableFeedURL = ComputeAzureArtifactsNuGetFeedUrl(feedName, account, project); + + packagesWithVersionsInReleaseFeeds[readableFeedURL] = await GetPackageVersionsForFeedAsync(account, project, feedName); + } + return packagesWithVersionsInReleaseFeeds; + } + + /// + /// Construct a nuget feed URL for an Azure DevOps Artifact feed + /// + /// Name of the feed + /// Azure DevOps Account where the feed is hosted + /// Optional project for the feed. + /// Url of the form https://pkgs.dev.azure.com/account/project/_packaging/feedName/nuget/v3/index.json + private static string ComputeAzureArtifactsNuGetFeedUrl(string feedName, string account, string project = "") + { + string projectSection = string.IsNullOrEmpty(project) ? "" : $"{project}/"; + return $"https://pkgs.dev.azure.com/{account}/{projectSection}_packaging/{feedName}/nuget/v3/index.json"; + } + + /// + /// Gets a Mapping of package -> versions for an Azure DevOps feed. + /// + /// Azure DevOps client. + /// Azure DevOps account. + /// Azure DevOps project the feed is hosted in. + /// Name of the feed + /// Dictionary where the key is the package name, and the value is a HashSet of the versions of the package in the feed + private async Task>> GetPackageVersionsForFeedAsync(string account, string project, string feedName) + { + var packagesWithVersions = new Dictionary>(); + List packagesInFeed = await _azureDevOpsClient.GetPackagesForFeedAsync(account, project, feedName); + foreach (AzureDevOpsPackage package in packagesInFeed) + { + packagesWithVersions.Add(package.Name, new HashSet(StringComparer.OrdinalIgnoreCase)); + packagesWithVersions[package.Name].UnionWith(package.Versions?.Where(v => !v.IsDeleted).Select(v => v.Version) ?? []); + } + return packagesWithVersions; + } + + /// + /// Updates the location for assets in the Database when + /// a version of an asset is found in the release feeds or in NuGet.org + /// + /// Feed to examine + /// Package to search for + /// Mapping of packages and their versions in the release feeds + /// Collection of versions that were updated for the package + private async Task> UpdateReleasedVersionsForPackageAsync( + AzureDevOpsFeed feed, + AzureDevOpsPackage package, + Dictionary>> dotnetFeedsPackageMapping) + { + var releasedVersions = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var version in package.Versions) + { + var matchingAssets = _context.Assets + .Include(a => a.Locations) + .Where(a => a.Name == package.Name && + a.Version == version.Version).AsEnumerable(); + + var matchingAsset = matchingAssets.FirstOrDefault( + a => a.Locations.Any(l => l.Location.Contains(feed.Name))); + + if (matchingAsset == null) + { + _logger.LogError($"Unable to find asset {package.Name}.{version.Version} in feed {feed.Name} in BAR. " + + $"Unable to determine if it was released or update its locations."); + continue; + } + else + { + if (matchingAsset.Locations.Any(l => l.Location == FeedConstants.NuGetOrgLocation || + dotnetFeedsPackageMapping.Any(f => l.Location == f.Key))) + { + _logger.LogInformation($"Package {package.Name}.{version.Version} is already present in a public location."); + releasedVersions.Add(version.Version); + } + else + { + List feedsWherePackageIsAvailable = GetReleaseFeedsWherePackageIsAvailable(package.Name, + version.Version, + dotnetFeedsPackageMapping); + + try + { + if (await IsPackageAvailableInNugetOrgAsync(package.Name, version.Version)) + { + feedsWherePackageIsAvailable.Add(FeedConstants.NuGetOrgLocation); + } + } + catch (HttpRequestException e) + { + _logger.LogInformation(e, $"Failed to determine if package {package.Name}.{version.Version} is present in NuGet.org"); + } + + + if (feedsWherePackageIsAvailable.Count > 0) + { + releasedVersions.Add(version.Version); + foreach (string feedToAdd in feedsWherePackageIsAvailable) + { + _logger.LogInformation($"Found package {package.Name}.{version.Version} in " + + $"{feedToAdd}, adding location to asset."); + + // TODO (https://github.com/dotnet/arcade-services/issues/3808) Don't actually do anything in BAR before we migrate fully to PCS + /*matchingAsset.Locations.Add(new AssetLocation() + { + Location = feedToAdd, + Type = LocationType.NugetFeed + }); + await _context.SaveChangesAsync();*/ + } + } + else + { + _logger.LogInformation($"Unable to find {package.Name}.{version} in any of the release feeds"); + } + } + } + } + return releasedVersions; + } + + /// + /// Deletes a version of a package from an Azure DevOps feed + /// + /// Feed to delete the package from + /// package to delete + /// Collection of versions to delete + /// + private async Task DeletePackageVersionsFromFeedAsync(AzureDevOpsFeed feed, string packageName, HashSet versionsToDelete) + { + foreach (string version in versionsToDelete) + { + try + { + _logger.LogInformation($"Deleting package {packageName}.{version} from feed {feed.Name}"); + + await _azureDevOpsClient.DeleteNuGetPackageVersionFromFeedAsync(feed.Account, + feed.Project?.Name, + feed.Name, + packageName, + version); + } + catch (HttpRequestException e) + { + _logger.LogError(e, $"There was an error attempting to delete package {packageName}.{version} from the {feed.Name} feed. Skipping..."); + } + } + } + + /// + /// Gets a list of feeds where a given package is available + /// + /// Package to search for + /// Version to search for + /// Feeds to search + /// List of feeds in the package mappings where the provided package and version are available + private List GetReleaseFeedsWherePackageIsAvailable( + string name, + string version, + Dictionary>> packageMappings) + { + List feeds = []; + foreach ((string feedName, Dictionary> packages) in packageMappings) + { + if (packages.TryGetValue(name, out HashSet? versions) && versions.Contains(version)) + { + feeds.Add(feedName); + } + } + + return feeds; + } + + /// + /// Checks whether a package is available in NuGet.org + /// by making a HEAD request to the package contents URI. + /// + /// Package to search for + /// Version to search for + /// True if the package is available in NuGet.org, false if not + private async Task IsPackageAvailableInNugetOrgAsync(string name, string version) + { + string packageContentsUri = $"{FeedConstants.NuGetOrgPackageBaseUrl}{name.ToLower()}/{version}/{name.ToLower()}.{version}.nupkg"; + try + { + using HttpRequestMessage headRequest = new(HttpMethod.Head, new Uri(packageContentsUri)); + using HttpResponseMessage response = await _httpClient.SendAsync(headRequest); + + response.EnsureSuccessStatusCode(); + _logger.LogInformation($"Found {name}.{version} in nuget.org URI: {packageContentsUri}"); + return true; + } + catch (HttpRequestException e) when (e.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) + { + _logger.LogInformation($"Unable to find {name}.{version} in nuget.org URI: {packageContentsUri}"); + return false; + } + } + + /// + /// Populates the packages and versions for a given feed + /// + /// Feed to populate + /// + private async Task PopulatePackagesForFeedAsync(AzureDevOpsFeed feed) + { + feed.Packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs new file mode 100644 index 0000000000..d94fd9e744 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerConfiguration.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Common.AzureDevOpsTokens; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ProductConstructionService.Common; + +namespace ProductConstructionService.FeedCleaner; + +public static class FeedCleanerConfiguration +{ + public static void ConfigureFeedCleaner(this IHostApplicationBuilder builder, ITelemetryChannel telemetryChannel) + { + builder.RegisterLogging(telemetryChannel); + builder.AddBuildAssetRegistry(); + + builder.Services.Configure((options, provider) => + { + builder.Configuration.GetSection("FeedCleaner").Bind(options); + + AzureDevOpsTokenProviderOptions azdoConfig = []; + builder.Configuration.GetSection("AzureDevOps").Bind(azdoConfig); + options.AzdoAccounts = azdoConfig.Keys.ToList(); + }); + + builder.Services.AddTransient(); + builder.Services.Configure("AzureDevOps", (o, s) => s.Bind(o)); + // TODO https://github.com/dotnet/arcade-services/issues/3808: + //builder.Services.AddTransient(); + builder.Services.AddTransient(); + builder.Services.AddTransient(sp => sp.GetRequiredService>()); + builder.Services.AddTransient(sp => ActivatorUtilities.CreateInstance(sp, "git")); + + builder.Services.AddTransient(); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs new file mode 100644 index 0000000000..4d186a01bc --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/FeedCleanerOptions.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace ProductConstructionService.FeedCleaner; + +public class FeedCleanerOptions +{ + public required bool Enabled { get; set; } + + public required List ReleasePackageFeeds { get; set; } + + public required List AzdoAccounts { get; set; } +} + +public record ReleasePackageFeed(string Account, string Project, string Name); diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj new file mode 100644 index 0000000000..018e1412c4 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj @@ -0,0 +1,28 @@ + + + + Exe + net8.0 + enable + enable + False + + + + + + + + + + + + + + + + PreserveNewest + PreserveNewest + + + diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/Program.cs b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/Program.cs new file mode 100644 index 0000000000..f4c03ace5a --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/Program.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.ApplicationInsights.Channel; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using ProductConstructionService.FeedCleaner; + +InMemoryChannel telemetryChannel = new(); +try +{ + var builder = Host.CreateApplicationBuilder(); + + builder.ConfigureFeedCleaner(telemetryChannel); + + // We're registering BAR context as a scoped service, so we have to create a scope to resolve it + var applicationScope = builder.Build().Services.CreateScope(); + + var cleaner = applicationScope.ServiceProvider.GetRequiredService(); + + await cleaner.CleanManagedFeedsAsync(); +} +finally +{ + telemetryChannel.Flush(); + await Task.Delay(TimeSpan.FromMilliseconds(1000)); +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Development.json new file mode 100644 index 0000000000..fccb226eb2 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "BuildAssetRegistrySqlConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true", + "AzureDevOps": { + "default": { + "UseLocalCredentials": true + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Staging.json new file mode 100644 index 0000000000..538acac674 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.Staging.json @@ -0,0 +1,9 @@ +{ + "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", + "AzureDevOps": { + "default": { + "ManagedIdentityId": "52833a45-ec68-4d75-8c83-e7df24649158" + } + }, + "ManagedIdentityClientId": "52833a45-ec68-4d75-8c83-e7df24649158" +} diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json new file mode 100644 index 0000000000..52d5401147 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/appsettings.json @@ -0,0 +1,27 @@ +{ + "FeedCleaner": { + "Enabled": false, + "ReleasePackageFeeds": [ + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet3" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet3.1" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet5" + }, + { + "Account": "dnceng", + "Project": "public", + "Name": "dotnet-tools" + } + ] + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs index d5c4015d1c..bcca41623a 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/LongestBuildPathUpdaterConfiguration.cs @@ -14,7 +14,7 @@ public static void ConfigureLongestBuildPathUpdater( this HostApplicationBuilder builder, ITelemetryChannel telemetryChannel) { - builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); + builder.RegisterLogging(telemetryChannel); builder.AddBuildAssetRegistry(); builder.Services.AddTransient(); diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs index 810f68231b..9bac5f153e 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/Program.cs @@ -1,6 +1,5 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Azure.Storage.Queues; using Maestro.Data.Models; using Microsoft.ApplicationInsights.Channel; using Microsoft.Extensions.Configuration; diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs index 5778fa9f72..675c594832 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggererConfiguration.cs @@ -26,7 +26,7 @@ public static HostApplicationBuilder ConfigureSubscriptionTriggerer( ManagedIdentityClientId = builder.Configuration[ProductConstructionServiceExtension.ManagedIdentityClientId] }); - builder.Services.RegisterLogging(telemetryChannel, builder.Environment.IsDevelopment()); + builder.RegisterLogging(telemetryChannel); builder.AddBuildAssetRegistry(); builder.AddWorkItemProducerFactory(credential); diff --git a/test/ProductConstructionService.FeedCleaner.Tests/DependencyRegistrationTest.cs b/test/ProductConstructionService.FeedCleaner.Tests/DependencyRegistrationTest.cs new file mode 100644 index 0000000000..8956bc2e2d --- /dev/null +++ b/test/ProductConstructionService.FeedCleaner.Tests/DependencyRegistrationTest.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.DotNet.Internal.DependencyInjection.Testing; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using NUnit.Framework; + +namespace ProductConstructionService.FeedCleaner.Tests; + +[TestFixture] +public class DependencyRegistrationTest +{ + [Test] + public void AreDependenciesRegistered() + { + var builder = Host.CreateApplicationBuilder(); + + builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "barConnectionString"; + + builder.ConfigureFeedCleaner(new InMemoryChannel()); + + DependencyInjectionValidation.IsDependencyResolutionCoherent(s => + { + foreach (var descriptor in builder.Services) + { + s.Add(descriptor); + } + }, + out var message, + additionalExemptTypes: [ + "Microsoft.Extensions.Hosting.ConsoleLifetimeOptions" + ]).Should().BeTrue(message); + } +} diff --git a/test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs b/test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs new file mode 100644 index 0000000000..5fae4b4909 --- /dev/null +++ b/test/ProductConstructionService.FeedCleaner.Tests/FeedCleanerTests.cs @@ -0,0 +1,311 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using FluentAssertions; +using Maestro.Common.AzureDevOpsTokens; +using Maestro.Data; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; +using NUnit.Framework; + +namespace ProductConstructionService.FeedCleaner.Tests; + +[TestFixture, NonParallelizable] +public class FeedCleanerTests +{ + private BuildAssetRegistryContext? _context; + private Mock _env = new(); + private ServiceProvider? _provider; + private IServiceScope _scope = new Mock().Object; + private Dictionary _feeds = []; + private FeedCleaner? _feedCleaner; + + private const string SomeAccount = "someAccount"; + private const string UnmanagedFeedName = "some-other-feed"; + private const string ReleaseFeedName = "release-feed"; + private const string FeedWithAllPackagesReleasedName = "darc-pub-some-repo-12345679"; + private const string FeedWithUnreleasedPackagesName = "darc-int-some-repo-12345678"; + + [SetUp] + public void FeedCleanerServiceTests_SetUp() + { + var services = new ServiceCollection(); + _env = new Mock(MockBehavior.Strict); + services.AddSingleton(_env.Object); + services.AddLogging(); + services.AddDbContext( + options => + { + options.UseInMemoryDatabase("BuildAssetRegistry"); + options.EnableServiceProviderCaching(false); + }); + services.Configure( + (options) => + { + options.Enabled = true; + options.ReleasePackageFeeds = + [ + new ReleasePackageFeed(SomeAccount, "someProject", ReleaseFeedName), + ]; + + options.AzdoAccounts = + [ + SomeAccount + ]; + } + ); + services.AddSingleton(); + _feeds = SetupFeeds(SomeAccount); + services.AddSingleton(SetupAzdoMock().Object); + services.Configure( + (options) => + { + options[SomeAccount] = new() + { + Token = "someToken" + }; + }); + + _provider = services.BuildServiceProvider(); + _scope = _provider.CreateScope(); + + _context = _scope.ServiceProvider.GetRequiredService(); + + SetupAssetsFromFeeds(); + _feedCleaner = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); + } + + [Test] + public async Task OnlyDeletesReleasedPackagesFromManagedFeeds() + { + await _feedCleaner!.CleanManagedFeedsAsync(); + var unreleasedFeed = _feeds[FeedWithUnreleasedPackagesName]; + unreleasedFeed.Packages.Should().HaveCount(2); + var packagesWithDeletedVersions = unreleasedFeed.Packages.Where(p => p.Versions.Any(v => v.IsDeleted)).ToList(); + packagesWithDeletedVersions.Should().ContainSingle(); + packagesWithDeletedVersions.First().Name.Should().Be("releasedPackage1"); + var deletedVersions = packagesWithDeletedVersions.First().Versions.Where(v => v.IsDeleted).ToList(); + deletedVersions.Should().ContainSingle(); + deletedVersions.First().Version.Should().Be("1.0"); + + _feeds[UnmanagedFeedName].Packages.Should().NotContain(p => p.Versions.Any(v => v.IsDeleted)); + } + + [Test] + [Ignore("(https://github.com/dotnet/arcade-services/issues/3808) ignore till this is resolved")] + public async Task UpdatesAssetLocationsForReleasedPackages() + { + await _feedCleaner!.CleanManagedFeedsAsync(); + + // Check the assets for the feed where all packages were released. + var assetsInDeletedFeed = _context!.Assets.Where(a => a.Locations.Any(l => l.Location.Contains(FeedWithAllPackagesReleasedName))).ToList(); + assetsInDeletedFeed.Should().HaveCount(4); + assetsInDeletedFeed.Should().Contain(a => a.Name.Equals("Newtonsoft.Json") && + a.Version == "12.0.2" && + a.Locations.Any(l => l.Location.Equals("https://api.nuget.org/v3/index.json"))); + + // All other assets should also have been updated to be in the release feed + assetsInDeletedFeed.Should().NotContain(a => !a.Name.Equals("Newtonsoft.Json") && + !a.Locations.Any(l => l.Location.Contains(ReleaseFeedName))); + + // "releasedPackage1" should've been released and have its location updated to the released feed. + var assetsInRemainingFeed = _context.Assets.Where(a => a.Locations.Any(l => l.Location.Contains(FeedWithUnreleasedPackagesName))).ToList(); + assetsInRemainingFeed.Should().HaveCount(2); + var releasedAssets = assetsInRemainingFeed.Where(a => a.Locations.Any(l => l.Location.Contains(ReleaseFeedName))).ToList(); + releasedAssets.Should().ContainSingle(); + releasedAssets.First().Name.Should().Be("releasedPackage1"); + releasedAssets.First().Version.Should().Be("1.0"); + + // "unreleasedPackage1" hasn't been released, should only have the stable feed as its location + var unreleasedAssets = assetsInRemainingFeed.Where(a => a.Locations.All(l => l.Location.Contains(FeedWithUnreleasedPackagesName))).ToList(); + unreleasedAssets.Should().ContainSingle(); + unreleasedAssets.First().Name.Should().Be("unreleasedPackage1"); + unreleasedAssets.First().Version.Should().Be("1.0"); + } + + private void SetupAssetsFromFeeds() + { + List assets = []; + foreach ((string feedName, AzureDevOpsFeed feed) in _feeds) + { + string projectSection = string.IsNullOrEmpty(feed.Project?.Name) ? "" : $"{feed.Project.Name}/"; + foreach (var package in feed.Packages) + { + foreach (var version in package.Versions) + { + assets.Add(new Asset() + { + Name = package.Name, + BuildId = 0, + NonShipping = false, + Version = version.Version, + Locations = + [ + new AssetLocation() + { + Type = LocationType.NugetFeed, + Location = $"https://pkgs.dev.azure.com/{feed.Account}/{projectSection}_packaging/{feedName}/nuget/v3/index.json" + } + ] + }); + } + + } + } + + _context!.Assets.AddRange(assets); + _context!.SaveChanges(); + } + + private Mock SetupAzdoMock() + { + var azdoClientMock = new Mock(MockBehavior.Strict); + azdoClientMock.Setup(a => a.GetFeedsAsync(SomeAccount)).ReturnsAsync(_feeds.Select(kvp => kvp.Value).ToList()); + azdoClientMock.Setup(a => a.GetPackagesForFeedAsync(SomeAccount, It.IsAny(), It.IsAny())) + .Returns((string account, string project, string feed) => Task.FromResult(_feeds[feed].Packages)); + azdoClientMock.Setup(a => a.DeleteNuGetPackageVersionFromFeedAsync(SomeAccount, It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((account, project, feed, package, version) => MarkVersionAsDeleted(_feeds[feed].Packages, package, version)) + .Returns(Task.CompletedTask); + azdoClientMock.Setup(a => a.DeleteFeedAsync(SomeAccount, It.IsAny(), It.IsAny())) + .Callback((account, project, feedIdentifier) => _feeds.Remove(feedIdentifier)) + .Returns(Task.CompletedTask); + return azdoClientMock; + } + + private static void MarkVersionAsDeleted(List packages, string packageName, string version) + { + foreach (var package in packages) + { + if (package.Name.Equals(packageName, StringComparison.OrdinalIgnoreCase)) + { + foreach (var packageVersion in package.Versions) + { + if (packageVersion.Version.Equals(version, StringComparison.OrdinalIgnoreCase)) + { + packageVersion.IsDeleted = true; + } + } + } + } + } + + private static Dictionary SetupFeeds(string account) + { + var someProject = new AzureDevOpsProject("0", "someProject"); + var allFeeds = new Dictionary(); + + // This is the reference release feed. + var releaseFeed = new AzureDevOpsFeed(account, "0", ReleaseFeedName, someProject) + { + Packages = + [ + new AzureDevOpsPackage("releasedPackage1", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false), + new AzureDevOpsPackageVersion("2.0", isDeleted: true), + ] + }, + new AzureDevOpsPackage("releasedPackage2", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false), + new AzureDevOpsPackageVersion("2.0", isDeleted: false), + ] + } + ] + }; + allFeeds.Add(releaseFeed.Name, releaseFeed); + + var managedFeedWithUnreleasedPackages = new AzureDevOpsFeed(account, "1", FeedWithUnreleasedPackagesName, null) + { + Packages = + [ + new AzureDevOpsPackage("unreleasedPackage1", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false) + ] + }, + new AzureDevOpsPackage("releasedPackage1", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false), + ] + } + ] + }; + allFeeds.Add(managedFeedWithUnreleasedPackages.Name, managedFeedWithUnreleasedPackages); + + var managedFeedWithEveryPackageReleased = new AzureDevOpsFeed(account, "2", FeedWithAllPackagesReleasedName, someProject) + { + Packages = + [ + new AzureDevOpsPackage("Newtonsoft.Json", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("12.0.2", isDeleted: false) + ] + }, + new AzureDevOpsPackage("releasedPackage1", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false) + ] + }, + new AzureDevOpsPackage("releasedPackage2", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false), + new AzureDevOpsPackageVersion("2.0", false) + ] + } + ] + }; + allFeeds.Add(managedFeedWithEveryPackageReleased.Name, managedFeedWithEveryPackageReleased); + + // add a feed with all released packages, but that doesn't match the name pattern. It shouldn't be touched by the cleaner. + var nonManagedFeedWithEveryPackageReleased = new AzureDevOpsFeed(account, "3", UnmanagedFeedName, someProject) + { + Packages = + [ + new AzureDevOpsPackage("Newtonsoft.Json", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("12.0.2", isDeleted: false) + ] + }, + new AzureDevOpsPackage("releasedPackage1", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false) + ] + }, + new AzureDevOpsPackage("releasedPackage2", "nuget") + { + Versions = + [ + new AzureDevOpsPackageVersion("1.0", isDeleted: false), + new AzureDevOpsPackageVersion("2.0", isDeleted: false) + ] + } + ] + }; + allFeeds.Add(nonManagedFeedWithEveryPackageReleased.Name, nonManagedFeedWithEveryPackageReleased); + + return allFeeds; + } +} diff --git a/test/ProductConstructionService.FeedCleaner.Tests/ProductConstructionService.FeedCleaner.Tests.csproj b/test/ProductConstructionService.FeedCleaner.Tests/ProductConstructionService.FeedCleaner.Tests.csproj new file mode 100644 index 0000000000..4176c6a671 --- /dev/null +++ b/test/ProductConstructionService.FeedCleaner.Tests/ProductConstructionService.FeedCleaner.Tests.csproj @@ -0,0 +1,21 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + From 07f6697cdbbf1ecbb18823241cbad7f241d5d98a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 29 Aug 2024 16:44:20 +0200 Subject: [PATCH 15/33] Load KV secrets into a dedicated config section (#3893) --- .../GitHubClientFactoryConfiguration.cs | 13 +++++++---- .../KeyVaultSecretsWithPrefix.cs | 15 +++++++++++++ .../PcsStartup.cs | 22 ++++++++++++++----- .../appsettings.json | 4 ---- 4 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Api/Configuration/KeyVaultSecretsWithPrefix.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/GitHubClientFactoryConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/GitHubClientFactoryConfiguration.cs index b0f40f1ef1..1ee182b11b 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/GitHubClientFactoryConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/GitHubClientFactoryConfiguration.cs @@ -8,9 +8,10 @@ namespace ProductConstructionService.Api.Configuration; public static class GitHubClientFactoryConfiguration { - private const string GitHubConfiguration = "GitHub"; - - public static void AddGitHubClientFactory(this WebApplicationBuilder builder) + public static void AddGitHubClientFactory( + this WebApplicationBuilder builder, + string? appId, + string? appSecret) { builder.Services.Configure(o => { @@ -22,6 +23,10 @@ public static void AddGitHubClientFactory(this WebApplicationBuilder builder) }); builder.Services.AddSingleton(); - builder.Services.Configure(builder.Configuration.GetSection(GitHubConfiguration)); + builder.Services.Configure(o => + { + o.GitHubAppId = !string.IsNullOrEmpty(appId) ? int.Parse(appId) : 0; + o.PrivateKey = appSecret; + }); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/KeyVaultSecretsWithPrefix.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/KeyVaultSecretsWithPrefix.cs new file mode 100644 index 0000000000..27754d0d14 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/KeyVaultSecretsWithPrefix.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Azure.Extensions.AspNetCore.Configuration.Secrets; +using Azure.Security.KeyVault.Secrets; + +namespace ProductConstructionService.Api.Configuration; + +internal class KeyVaultSecretsWithPrefix(string prefix) : KeyVaultSecretManager +{ + private readonly string _prefix = prefix; + + public override string GetKey(KeyVaultSecret secret) + => _prefix + secret.Name.Replace("--", ConfigurationPath.KeyDelimiter); +} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 3eec5f92df..5a0c06c651 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -36,7 +36,6 @@ using ProductConstructionService.Api.VirtualMonoRepo; using ProductConstructionService.Common; using ProductConstructionService.DependencyFlow.WorkItems; -using ProductConstructionService.DependencyFlow.WorkItemProcessors; using ProductConstructionService.WorkItems; using ProductConstructionService.DependencyFlow; @@ -48,13 +47,19 @@ internal static class PcsStartup private static class ConfigurationKeys { + // All secrets loaded from KeyVault will have this prefix + public const string KeyVaultSecretPrefix = "KeyVaultSecrets:"; + + // Secrets coming from the KeyVault + public const string GitHubClientId = $"{KeyVaultSecretPrefix}github-app-id"; + public const string GitHubClientSecret = $"{KeyVaultSecretPrefix}github-app-private-key"; + public const string GitHubToken = $"{KeyVaultSecretPrefix}BotAccount-dotnet-bot-repo-PAT"; + + // Configuration from appsettings.json public const string AzureDevOpsConfiguration = "AzureDevOps"; public const string DatabaseConnectionString = "BuildAssetRegistrySqlConnectionString"; public const string DependencyFlowSLAs = "DependencyFlowSLAs"; public const string EntraAuthenticationKey = "EntraAuthentication"; - public const string GitHubClientId = "github-oauth-id"; - public const string GitHubClientSecret = "github-oauth-secret"; - public const string GitHubToken = "BotAccount-dotnet-bot-repo-PAT"; public const string KeyVaultName = "KeyVaultName"; public const string ManagedIdentityId = "ManagedIdentityClientId"; } @@ -158,7 +163,10 @@ internal static async Task ConfigurePcs( if (addKeyVault) { Uri keyVaultUri = new($"https://{builder.Configuration.GetRequiredValue(ConfigurationKeys.KeyVaultName)}.vault.azure.net/"); - builder.Configuration.AddAzureKeyVault(keyVaultUri, azureCredential); + builder.Configuration.AddAzureKeyVault( + keyVaultUri, + azureCredential, + new KeyVaultSecretsWithPrefix(ConfigurationKeys.KeyVaultSecretPrefix)); } // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator @@ -169,7 +177,9 @@ internal static async Task ConfigurePcs( builder.AddDependencyFlowProcessors(); builder.AddVmrRegistrations(gitHubToken); builder.AddMaestroApiClient(managedIdentityId); - builder.AddGitHubClientFactory(); + builder.AddGitHubClientFactory( + builder.Configuration[ConfigurationKeys.GitHubClientId], + builder.Configuration[ConfigurationKeys.GitHubClientSecret]); builder.Services.AddGitHubTokenProvider(); builder.Services.AddScoped(); builder.Services.AddSingleton(); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json index 8e9d94ffd3..69b13a58f3 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json @@ -1,8 +1,4 @@ { - "GitHub": { - "GitHubAppId": "[vault(github-app-id)]", - "PrivateKey": "[vault(github-app-private-key)]" - }, "Logging": { "LogLevel": { "Default": "Information", From 3ef7530e8b393d43187e894e561dcf89490bc605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 29 Aug 2024 18:13:38 +0200 Subject: [PATCH 16/33] Use unique branch/channel names per each test (#3895) --- .../MaestroScenarioTestBase.cs | 11 ++++++++ .../ScenarioTests_AzDoFlow.cs | 16 ++++++------ .../ScenarioTests_Channels.cs | 3 +-- .../ScenarioTests_DefaultChannels.cs | 6 ++--- .../ScenarioTests_Dependencies.cs | 7 +++-- .../ScenarioTests_GitHubFlow.cs | 20 +++++++------- .../ScenarioTests_RepoPolicies.cs | 26 +++++++++---------- .../ScenarioTests_Subscriptions.cs | 6 ++--- .../ScenarioTestBase.cs | 11 ++++++++ .../ScenarioTests_AzDoFlow.cs | 16 ++++++------ .../ScenarioTests_Channels.cs | 2 +- .../ScenarioTests_DefaultChannels.cs | 6 ++--- .../ScenarioTests_Dependencies.cs | 6 ++--- .../ScenarioTests_GitHubFlow.cs | 20 +++++++------- .../ScenarioTests_RepoPolicies.cs | 25 +++++++++--------- .../ScenarioTests_Subscriptions.cs | 7 +++-- 16 files changed, 104 insertions(+), 84 deletions(-) diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index 3a0a98cbed..d08bbc11ab 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; @@ -949,4 +950,14 @@ protected async Task ValidateGithubMaestroCheckRunsSuccessful(string targe return true; } + + protected static string GetTestChannelName([CallerMemberName] string testName = "") + { + return $"c{testName}_{Guid.NewGuid().ToString().Substring(0, 16)}"; + } + + protected static string GetTestBranchName([CallerMemberName] string testName = "") + { + return $"b{testName}_{Guid.NewGuid().ToString().Substring(0, 16)}"; + } } diff --git a/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs b/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs index 9b2e29936c..fe5d4bec35 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_AzDoFlow.cs @@ -112,8 +112,8 @@ public async Task Darc_AzDoFlow_Batched() var expectedDependencies = _expectedAzDoDependenciesSource1.Concat(_expectedAzDoDependenciesSource2).ToList(); await testLogic.DarcBatchedFlowTestBase( - $"AzDo_BatchedTestBranch_{Environment.MachineName}", - $"AzDo Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source2Assets, expectedDependencies, @@ -130,8 +130,8 @@ public async Task Darc_AzDoFlow_NonBatched_AllChecksSuccessful() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedAzDoFlowTestBase( - $"AzDo_NonBatchedTestBranch_AllChecks_{Environment.MachineName}", - $"AzDo Non-Batched All Checks Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _expectedAzDoDependenciesSource1, allChecks: true).ConfigureAwait(true); @@ -147,8 +147,8 @@ public async Task Darc_AzDoFlow_NonBatched() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedUpdatingAzDoFlowTestBase( - $"AzDo_NonBatchedTestBranch_{Environment.MachineName}", - $"AzDo Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source1AssetsUpdated, _expectedAzDoDependenciesSource1, @@ -255,8 +255,8 @@ public async Task Darc_AzDoFlow_FeedFlow() }; expectedAzDoFeedFlowDependencies.Add(feedHamburger); await testLogic.NonBatchedAzDoFlowTestBase( - $"AzDo_FeedFlowBranch_{Environment.MachineName}", - $"AzDo_FeedFlowChannel_{Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), feedFlowSourceAssets, expectedAzDoFeedFlowDependencies, isFeedTest: true, diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs b/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs index 6e60dafa7a..d4a8e6ff9c 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Channels.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Threading.Tasks; using FluentAssertions; using NUnit.Framework; @@ -29,7 +28,7 @@ public async Task ArcadeChannels_EndToEnd() SetTestParameters(_parameters); // Create a new channel - string testChannelName = $"Test Channel End to End {Environment.MachineName}"; + string testChannelName = GetTestChannelName(); await using (AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false)) { diff --git a/test/Maestro.ScenarioTests/ScenarioTests_DefaultChannels.cs b/test/Maestro.ScenarioTests/ScenarioTests_DefaultChannels.cs index 10eabc68fb..79af4960e1 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_DefaultChannels.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_DefaultChannels.cs @@ -20,7 +20,7 @@ internal class ScenarioTests_DefaultChannels : MaestroScenarioTestBase public ScenarioTests_DefaultChannels() { - _branchName = $"ChannelTestBranch_{Environment.MachineName}"; + _branchName = GetTestBranchName(); _branchNameWithRefsHeads = $"refs/heads/{_branchName}"; } @@ -43,8 +43,8 @@ public async Task ArcadeChannels_DefaultChannels() { string repoUrl = GetGitHubRepoUrl(_repoName); - string testChannelName1 = $"TestChannelDefault1_{Environment.MachineName}"; - string testChannelName2 = $"TestChannelDefault2_{Environment.MachineName}"; + string testChannelName1 = GetTestChannelName(); + string testChannelName2 = GetTestChannelName(); await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(testChannelName1)) { diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs b/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs index 6f7542ef8d..b96d01f5eb 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Dependencies.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Collections.Immutable; using System.Threading.Tasks; using FluentAssertions; @@ -43,9 +42,9 @@ public async Task ArcadeDependencies_EndToEnd() var sourceBuildNumber = "654321"; var sourceCommit = "SourceCommitVar"; var targetCommit = "TargetCommitVar"; - var sourceBranch = $"DependenciesSourceBranch_{Environment.MachineName}"; - var targetBranch = $"DependenciesTargetBranch_{Environment.MachineName}"; - var testChannelName = $"TestChannel_Dependencies_{Environment.MachineName}"; + var sourceBranch = GetTestBranchName(); + var targetBranch = GetTestBranchName(); + var testChannelName = GetTestChannelName(); IImmutableList source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); IImmutableList source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0"); diff --git a/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs b/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs index 0355091a4e..ab11e1594d 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_GitHubFlow.cs @@ -115,8 +115,8 @@ public async Task Darc_GitHubFlow_Batched() var expectedDependencies = _expectedDependenciesSource1.Concat(_expectedDependenciesSource2).ToList(); await testLogic.DarcBatchedFlowTestBase( - $"GitHub_BatchedTestBranch_{Environment.MachineName}", - $"GitHub Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source2Assets, expectedDependencies, @@ -132,8 +132,8 @@ public async Task Darc_GitHubFlow_NonBatched() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedUpdatingGitHubFlowTestBase( - $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", - $"GitHub Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source1AssetsUpdated, _expectedDependenciesSource1, @@ -173,8 +173,8 @@ public async Task Darc_GitHubFlow_NonBatched_StrictCoherency() IImmutableList sourceAssets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); await testLogic.NonBatchedGitHubFlowTestBase( - $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", - $"GitHub Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, expectedCoherencyDependencies, allChecks: true).ConfigureAwait(false); @@ -230,8 +230,8 @@ public async Task Darc_GitHubFlow_NonBatched_FailingCoherencyUpdate() IImmutableList childSourceAssets = GetAssetData("Fzz", "1.1.0", "ASD", "1.1.1"); await testLogic.NonBatchedGitHubFlowCoherencyTestBase( - $"GitHub_NonBatchedTestBranch_FailingCoherencyUpdate_{Environment.MachineName}", - $"GitHub Non-Batched Channel FailingCoherencyUpdate {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, childSourceAssets, expectedCoherencyDependencies, @@ -311,8 +311,8 @@ public async Task Darc_GitHubFlow_NonBatched_FailingCoherentOnlyUpdate() IImmutableList childSourceAssets = GetAssetData("B1", "2.1.0", "B2", "2.1.0"); await testLogic.NonBatchedGitHubFlowCoherencyOnlyTestBase( - $"GitHub_NonBatchedTestBranch_FailingCoherencyOnlyUpdate_{Environment.MachineName}", - $"GitHub Non-Batched Channel FailingCoherencyOnlyUpdate {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, childSourceAssets, expectedNonCoherencyDependencies, diff --git a/test/Maestro.ScenarioTests/ScenarioTests_RepoPolicies.cs b/test/Maestro.ScenarioTests/ScenarioTests_RepoPolicies.cs index 2b53dbb5f2..7d2bc4499a 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_RepoPolicies.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_RepoPolicies.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Threading.Tasks; using FluentAssertions; using NUnit.Framework; @@ -14,9 +13,6 @@ namespace Maestro.ScenarioTests; [Parallelizable] internal class ScenarioTests_RepoPolicies : MaestroScenarioTestBase { - // The RepoPolicies logic does a partial string match for the branch name in the base, - // so it's important that this branch name not be a substring or superstring of another branch name - private readonly string _branchName = $"MaestroRepoPoliciesTestBranch_{Environment.MachineName}"; private readonly string _repoName = TestRepository.TestRepo1Name; private TestParameters _parameters; @@ -38,22 +34,26 @@ public async Task ArcadeRepoPolicies_EndToEnd() string repoUrl = GetGitHubRepoUrl(_repoName); + // The RepoPolicies logic does a partial string match for the branch name in the base, + // so it's important that this branch name not be a substring or superstring of another branch name + var branchName = GetTestBranchName(); + TestContext.WriteLine("Setting repository merge policy to empty"); - await SetRepositoryPolicies(repoUrl, _branchName); - string emptyPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - string expectedEmpty = $"{repoUrl} @ {_branchName}\r\n- Merge Policies: []\r\n"; + await SetRepositoryPolicies(repoUrl, branchName); + string emptyPolicies = await GetRepositoryPolicies(repoUrl, branchName); + string expectedEmpty = $"{repoUrl} @ {branchName}\r\n- Merge Policies: []\r\n"; emptyPolicies.Should().BeEquivalentTo(expectedEmpty, "Repository merge policy is not empty"); TestContext.WriteLine("Setting repository merge policy to standard"); - await SetRepositoryPolicies(repoUrl, _branchName, ["--standard-automerge"]); - string standardPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - string expectedStandard = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n Standard\r\n"; + await SetRepositoryPolicies(repoUrl, branchName, ["--standard-automerge"]); + string standardPolicies = await GetRepositoryPolicies(repoUrl, branchName); + string expectedStandard = $"{repoUrl} @ {branchName}\r\n- Merge Policies:\r\n Standard\r\n"; standardPolicies.Should().BeEquivalentTo(expectedStandard, "Repository policy not set to standard"); TestContext.WriteLine("Setting repository merge policy to all checks successful"); - await SetRepositoryPolicies(repoUrl, _branchName, ["--all-checks-passed", "--ignore-checks", "A,B"]); - string allChecksPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - string expectedAllChecksPolicies = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n AllChecksSuccessful\r\n ignoreChecks = \r\n" + + await SetRepositoryPolicies(repoUrl, branchName, ["--all-checks-passed", "--ignore-checks", "A,B"]); + string allChecksPolicies = await GetRepositoryPolicies(repoUrl, branchName); + string expectedAllChecksPolicies = $"{repoUrl} @ {branchName}\r\n- Merge Policies:\r\n AllChecksSuccessful\r\n ignoreChecks = \r\n" + " [\r\n" + " \"A\",\r\n" + " \"B\"\r\n" + diff --git a/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs b/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs index a41d191a7a..5619ec2abf 100644 --- a/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs +++ b/test/Maestro.ScenarioTests/ScenarioTests_Subscriptions.cs @@ -34,8 +34,8 @@ public async Task Subscriptions_EndToEnd() TestContext.WriteLine("Subscription management tests..."); string repo1Name = TestRepository.TestRepo1Name; string repo2Name = TestRepository.TestRepo2Name; - string channel1Name = $"SubscriptionEndToEnd_TestChannel1_{Environment.MachineName}"; - string channel2Name = $"SubscriptionEndToEnd_TestChannel2_{Environment.MachineName}"; + string channel1Name = GetTestChannelName(); + string channel2Name = GetTestChannelName(); _parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); SetTestParameters(_parameters); @@ -43,7 +43,7 @@ public async Task Subscriptions_EndToEnd() string repo1Uri = GetGitHubRepoUrl(repo1Name); string repo2Uri = GetGitHubRepoUrl(repo2Name); string repo1AzDoUri = GetAzDoRepoUrl(repo1Name); - string targetBranch = $"SubscriptionEndToEnd_TargetBranch_{Environment.MachineName}"; + string targetBranch = GetTestBranchName(); TestContext.WriteLine($"Creating channels {channel1Name} and {channel2Name}"); await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(channel1Name).ConfigureAwait(false)) diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs index bfc62caee6..b911b2e19f 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Immutable; +using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; using FluentAssertions; @@ -943,4 +944,14 @@ protected async Task ValidateGithubMaestroCheckRunsSuccessful(string targe return true; } + + protected static string GetTestChannelName([CallerMemberName] string testName = "") + { + return $"c{testName}_{Guid.NewGuid().ToString().Substring(0, 16)}"; + } + + protected static string GetTestBranchName([CallerMemberName] string testName = "") + { + return $"b{testName}_{Guid.NewGuid().ToString().Substring(0, 16)}"; + } } diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs index 63d7c3f95c..e89ecf5a5e 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_AzDoFlow.cs @@ -108,8 +108,8 @@ public async Task Darc_AzDoFlow_Batched() var expectedDependencies = _expectedAzDoDependenciesSource1.Concat(_expectedAzDoDependenciesSource2).ToList(); await testLogic.DarcBatchedFlowTestBase( - $"AzDo_BatchedTestBranch_{Environment.MachineName}", - $"AzDo Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source2Assets, expectedDependencies, @@ -126,8 +126,8 @@ public async Task Darc_AzDoFlow_NonBatched_AllChecksSuccessful() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedAzDoFlowTestBase( - $"AzDo_NonBatchedTestBranch_AllChecks_{Environment.MachineName}", - $"AzDo Non-Batched All Checks Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _expectedAzDoDependenciesSource1, allChecks: true).ConfigureAwait(true); @@ -143,8 +143,8 @@ public async Task Darc_AzDoFlow_NonBatched() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedUpdatingAzDoFlowTestBase( - $"AzDo_NonBatchedTestBranch_{Environment.MachineName}", - $"AzDo Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source1AssetsUpdated, _expectedAzDoDependenciesSource1, @@ -251,8 +251,8 @@ public async Task Darc_AzDoFlow_FeedFlow() }; expectedAzDoFeedFlowDependencies.Add(feedHamburger); await testLogic.NonBatchedAzDoFlowTestBase( - $"AzDo_FeedFlowBranch_{Environment.MachineName}", - $"AzDo_FeedFlowChannel_{Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), feedFlowSourceAssets, expectedAzDoFeedFlowDependencies, isFeedTest: true, diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs index be36407b60..f8110292e3 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Channels.cs @@ -27,7 +27,7 @@ public async Task ArcadeChannels_EndToEnd() SetTestParameters(_parameters); // Create a new channel - var testChannelName = $"Test Channel End to End {Environment.MachineName}"; + var testChannelName = GetTestChannelName(); await using (AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName).ConfigureAwait(false)) { diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs index 83689de3e9..8b322591d1 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_DefaultChannels.cs @@ -18,7 +18,7 @@ internal class ScenarioTests_DefaultChannels : ScenarioTestBase public ScenarioTests_DefaultChannels() { - _branchName = $"ChannelTestBranch_{Environment.MachineName}"; + _branchName = GetTestBranchName(); _branchNameWithRefsHeads = $"refs/heads/{_branchName}"; } @@ -41,8 +41,8 @@ public async Task ArcadeChannels_DefaultChannels() { var repoUrl = GetGitHubRepoUrl(_repoName); - var testChannelName1 = $"TestChannelDefault1_{Environment.MachineName}"; - var testChannelName2 = $"TestChannelDefault2_{Environment.MachineName}"; + var testChannelName1 = GetTestChannelName(); + var testChannelName2 = GetTestChannelName(); await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(testChannelName1)) { diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs index 2014d20a82..6ee38738e9 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Dependencies.cs @@ -41,9 +41,9 @@ public async Task ArcadeDependencies_EndToEnd() var sourceBuildNumber = "654321"; var sourceCommit = "SourceCommitVar"; var targetCommit = "TargetCommitVar"; - var sourceBranch = $"DependenciesSourceBranch_{Environment.MachineName}"; - var targetBranch = $"DependenciesTargetBranch_{Environment.MachineName}"; - var testChannelName = $"TestChannel_Dependencies_{Environment.MachineName}"; + var sourceBranch = GetTestBranchName(); + var targetBranch = GetTestBranchName(); + var testChannelName = GetTestChannelName(); IImmutableList source1Assets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); IImmutableList source2Assets = GetAssetData("Pizza", "3.1.0", "Hamburger", "4.1.0"); diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs index 4d6e60ded9..3966f2f2fb 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_GitHubFlow.cs @@ -111,8 +111,8 @@ public async Task Darc_GitHubFlow_Batched() var expectedDependencies = _expectedDependenciesSource1.Concat(_expectedDependenciesSource2).ToList(); await testLogic.DarcBatchedFlowTestBase( - $"GitHub_BatchedTestBranch_{Environment.MachineName}", - $"GitHub Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source2Assets, expectedDependencies, @@ -128,8 +128,8 @@ public async Task Darc_GitHubFlow_NonBatched() var testLogic = new EndToEndFlowLogic(parameters); await testLogic.NonBatchedUpdatingGitHubFlowTestBase( - $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", - $"GitHub Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), _source1Assets, _source1AssetsUpdated, _expectedDependenciesSource1, @@ -169,8 +169,8 @@ public async Task Darc_GitHubFlow_NonBatched_StrictCoherency() IImmutableList sourceAssets = GetAssetData("Foo", "1.1.0", "Bar", "2.1.0"); await testLogic.NonBatchedGitHubFlowTestBase( - $"GitHub_NonBatchedTestBranch_{Environment.MachineName}", - $"GitHub Non-Batched Channel {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, expectedCoherencyDependencies, allChecks: true).ConfigureAwait(false); @@ -226,8 +226,8 @@ public async Task Darc_GitHubFlow_NonBatched_FailingCoherencyUpdate() IImmutableList childSourceAssets = GetAssetData("Fzz", "1.1.0", "ASD", "1.1.1"); await testLogic.NonBatchedGitHubFlowCoherencyTestBase( - $"GitHub_NonBatchedTestBranch_FailingCoherencyUpdate_{Environment.MachineName}", - $"GitHub Non-Batched Channel FailingCoherencyUpdate {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, childSourceAssets, expectedCoherencyDependencies, @@ -307,8 +307,8 @@ public async Task Darc_GitHubFlow_NonBatched_FailingCoherentOnlyUpdate() IImmutableList childSourceAssets = GetAssetData("B1", "2.1.0", "B2", "2.1.0"); await testLogic.NonBatchedGitHubFlowCoherencyOnlyTestBase( - $"GitHub_NonBatchedTestBranch_FailingCoherencyOnlyUpdate_{Environment.MachineName}", - $"GitHub Non-Batched Channel FailingCoherencyOnlyUpdate {Environment.MachineName}", + GetTestBranchName(), + GetTestChannelName(), sourceAssets, childSourceAssets, expectedNonCoherencyDependencies, diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs index 80c05bc637..8e19f05f13 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_RepoPolicies.cs @@ -12,9 +12,6 @@ namespace ProductConstructionService.ScenarioTests; [Parallelizable] internal class ScenarioTests_RepoPolicies : ScenarioTestBase { - // The RepoPolicies logic does a partial string match for the branch name in the base, - // so it's important that this branch name not be a substring or superstring of another branch name - private readonly string _branchName = $"MaestroRepoPoliciesTestBranch_{Environment.MachineName}"; private readonly string _repoName = TestRepository.TestRepo1Name; private TestParameters _parameters; @@ -36,22 +33,26 @@ public async Task ArcadeRepoPolicies_EndToEnd() var repoUrl = GetGitHubRepoUrl(_repoName); + // The RepoPolicies logic does a partial string match for the branch name in the base, + // so it's important that this branch name not be a substring or superstring of another branch name + var branchName = GetTestBranchName(); + TestContext.WriteLine("Setting repository merge policy to empty"); - await SetRepositoryPolicies(repoUrl, _branchName); - var emptyPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - var expectedEmpty = $"{repoUrl} @ {_branchName}\r\n- Merge Policies: []\r\n"; + await SetRepositoryPolicies(repoUrl, branchName); + var emptyPolicies = await GetRepositoryPolicies(repoUrl, branchName); + var expectedEmpty = $"{repoUrl} @ {branchName}\r\n- Merge Policies: []\r\n"; emptyPolicies.Should().BeEquivalentTo(expectedEmpty, "Repository merge policy is not empty"); TestContext.WriteLine("Setting repository merge policy to standard"); - await SetRepositoryPolicies(repoUrl, _branchName, ["--standard-automerge"]); - var standardPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - var expectedStandard = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n Standard\r\n"; + await SetRepositoryPolicies(repoUrl, branchName, ["--standard-automerge"]); + var standardPolicies = await GetRepositoryPolicies(repoUrl, branchName); + var expectedStandard = $"{repoUrl} @ {branchName}\r\n- Merge Policies:\r\n Standard\r\n"; standardPolicies.Should().BeEquivalentTo(expectedStandard, "Repository policy not set to standard"); TestContext.WriteLine("Setting repository merge policy to all checks successful"); - await SetRepositoryPolicies(repoUrl, _branchName, ["--all-checks-passed", "--ignore-checks", "A,B"]); - var allChecksPolicies = await GetRepositoryPolicies(repoUrl, _branchName); - var expectedAllChecksPolicies = $"{repoUrl} @ {_branchName}\r\n- Merge Policies:\r\n AllChecksSuccessful\r\n ignoreChecks = \r\n" + + await SetRepositoryPolicies(repoUrl, branchName, ["--all-checks-passed", "--ignore-checks", "A,B"]); + var allChecksPolicies = await GetRepositoryPolicies(repoUrl, branchName); + var expectedAllChecksPolicies = $"{repoUrl} @ {branchName}\r\n- Merge Policies:\r\n AllChecksSuccessful\r\n ignoreChecks = \r\n" + " [\r\n" + " \"A\",\r\n" + " \"B\"\r\n" + diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs index a839cb07de..24f8a670bd 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests_Subscriptions.cs @@ -6,7 +6,6 @@ using Microsoft.DotNet.Darc; using NUnit.Framework; using NUnit.Framework.Internal; -using ProductConstructionService.Client.Models; using ProductConstructionService.ScenarioTests.ObjectHelpers; namespace ProductConstructionService.ScenarioTests; @@ -32,8 +31,8 @@ public async Task Subscriptions_EndToEnd() TestContext.WriteLine("Subscription management tests..."); var repo1Name = TestRepository.TestRepo1Name; var repo2Name = TestRepository.TestRepo2Name; - var channel1Name = $"SubscriptionEndToEnd_TestChannel1_{Environment.MachineName}"; - var channel2Name = $"SubscriptionEndToEnd_TestChannel2_{Environment.MachineName}"; + var channel1Name = GetTestChannelName(); + var channel2Name = GetTestChannelName(); _parameters = await TestParameters.GetAsync(useNonPrimaryEndpoint: true); SetTestParameters(_parameters); @@ -41,7 +40,7 @@ public async Task Subscriptions_EndToEnd() var repo1Uri = GetGitHubRepoUrl(repo1Name); var repo2Uri = GetGitHubRepoUrl(repo2Name); var repo1AzDoUri = GetAzDoRepoUrl(repo1Name); - var targetBranch = $"SubscriptionEndToEnd_TargetBranch_{Environment.MachineName}"; + var targetBranch = GetTestBranchName(); TestContext.WriteLine($"Creating channels {channel1Name} and {channel2Name}"); await using (AsyncDisposableValue channel1 = await CreateTestChannelAsync(channel1Name).ConfigureAwait(false)) From e9cb4bd5ac05cf3b9203402f1d281b79bb4b869f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 10:02:55 +0200 Subject: [PATCH 17/33] Update Aspire to 8.2.0 (#3896) --- Directory.Packages.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 592c06a4de..d8479a1a97 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,7 +4,7 @@ 3.1.24 6.0.26 6.0.0 - 8.1.0 + 8.2.0 true true @@ -18,7 +18,7 @@ - + @@ -150,6 +150,6 @@ - + From 4e7aefcf1dd5e3ad1380944b97e9e80a10859dfe Mon Sep 17 00:00:00 2001 From: Stephen Halter Date: Fri, 30 Aug 2024 01:33:36 -0700 Subject: [PATCH 18/33] Use BuildDependency.DependentBuildId over BuildId in Incoming.cshtml.cs (#3897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Přemek Vysoký --- .../Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs b/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs index a70a7049c3..699b13f777 100644 --- a/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs +++ b/src/Maestro/Maestro.Web/Pages/DependencyFlow/Incoming.cshtml.cs @@ -84,11 +84,11 @@ public async Task OnGet(int channelId, string owner, string repo) var incoming = new List(); foreach (var dep in Build.DependentBuildIds) { - var lastConsumedBuildOfDependency = graph[dep.BuildId]; + var lastConsumedBuildOfDependency = graph[dep.DependentBuildId]; if (lastConsumedBuildOfDependency == null) { - _logger.LogWarning("Failed to find build with id '{BuildId}' in the graph", dep.BuildId); + _logger.LogWarning("Failed to find build with id '{DependentBuildId}' in the graph", dep.DependentBuildId); continue; } From 116431ddeeb7d8e12d85baf18b353bc7c6cced15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 10:41:59 +0200 Subject: [PATCH 19/33] Hardcode Redis port in local devenv (#3900) --- .../ProductConstructionService.AppHost/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ProductConstructionService/ProductConstructionService.AppHost/Program.cs b/src/ProductConstructionService/ProductConstructionService.AppHost/Program.cs index 96b33497cf..26202e1933 100644 --- a/src/ProductConstructionService/ProductConstructionService.AppHost/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.AppHost/Program.cs @@ -3,7 +3,7 @@ var builder = DistributedApplication.CreateBuilder(args); -var redisCache = builder.AddRedis("redis"); +var redisCache = builder.AddRedis("redis", port: 55689); var queues = builder.AddAzureStorage("storage") .RunAsEmulator(emulator => emulator.WithImageTag("3.31.0")) // Workaround for https://github.com/dotnet/aspire/issues/5078 .AddQueues("queues"); From 65bc661685d8b780ad7944a6d67a0156561b34e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 11:08:40 +0200 Subject: [PATCH 20/33] Make API 404s contain JSON body (#3902) --- .../ProductConstructionService.Api/Pages/Error.cshtml.cs | 4 ---- .../ProductConstructionService.Api/PcsStartup.cs | 2 +- .../ProductConstructionService.Api/Program.cs | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Pages/Error.cshtml.cs b/src/ProductConstructionService/ProductConstructionService.Api/Pages/Error.cshtml.cs index bee6ee6a84..2f0e73c027 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Pages/Error.cshtml.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Pages/Error.cshtml.cs @@ -1,10 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. - -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - using Microsoft.AspNetCore.Mvc.RazorPages; namespace ProductConstructionService.Api.Pages; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 5a0c06c651..1ef3b05434 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -282,6 +282,7 @@ internal static async Task ConfigurePcs( public static void ConfigureApi(this IApplicationBuilder app, bool isDevelopment) { + app.UseApiRedirection(); app.UseExceptionHandler(a => a.Run(async ctx => { @@ -292,7 +293,6 @@ public static void ConfigureApi(this IApplicationBuilder app, bool isDevelopment ctx.Response.StatusCode = (int)HttpStatusCode.InternalServerError; await ctx.Response.WriteAsync(output, Encoding.UTF8); })); - app.UseApiRedirection(); app.UseEndpoints(e => { var controllers = e.MapControllers(); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs index 70f377e1ca..19f9bf70c6 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Program.cs @@ -67,7 +67,6 @@ await app.Services.UseLocalWorkItemQueues( app.UseStaticFiles(); } -app.UseStatusCodePagesWithReExecute("/Error", "?code={0}"); app.UseCookiePolicy(); app.UseAuthentication(); app.UseRouting(); @@ -79,6 +78,7 @@ await app.Services.UseLocalWorkItemQueues( a => PcsStartup.ConfigureApi(a, isDevelopment)); // Add security headers +app.UseStatusCodePagesWithReExecute("/Error", "?code={0}"); app.ConfigureSecurityHeaders(); // Map pages and non-API controllers From de9421cd22a21160726a6035d61a73b4a1e5bdec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 11:45:49 +0200 Subject: [PATCH 21/33] Batch PCS pipeline runs (#3904) --- azure-pipelines-product-construction-service.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-product-construction-service.yml b/azure-pipelines-product-construction-service.yml index c98f6d24e3..16bad7e91a 100644 --- a/azure-pipelines-product-construction-service.yml +++ b/azure-pipelines-product-construction-service.yml @@ -2,6 +2,7 @@ name: $(Date:yyyyMMdd)$(Rev:r) trigger: + batch: true branches: include: - main From 25e1905ff780d9650862a3a145b9ad684ac5a8a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 12:08:46 +0200 Subject: [PATCH 22/33] Fix DLL path in container jobs (#3903) --- .../ProductConstructionService/provision.bicep | 8 ++++---- .../SubscriptionTriggerer.cs | 5 +++-- .../ApiTestConfiguration.cs | 2 -- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 029a56963b..c552213a2d 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -600,7 +600,7 @@ module subscriptionTriggererTwiceDaily 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/SubscriptionTriggerer.dll' + dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' argument: 'twicedaily' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId @@ -622,7 +622,7 @@ module subscriptionTriggererDaily 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/SubscriptionTriggerer.dll' + dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' argument: 'daily' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId @@ -644,7 +644,7 @@ module subscriptionTriggererWeekly 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/SubscriptionTriggerer/SubscriptionTriggerer.dll' + dllFullPath: '/app/SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.dll' argument: 'weekly' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId @@ -666,7 +666,7 @@ module longestBuildPathUpdater 'scheduledContainerJob.bicep' = { containerRegistryName: containerRegistryName containerAppsEnvironmentId: containerAppsEnvironment.id containerImageName: containerImageName - dllFullPath: '/app/LongestBuildPathUpdater/LongestBuildPathUpdater.dll' + dllFullPath: '/app/LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.dll' argument: '' contributorRoleId: contributorRole deploymentIdentityPrincipalId: deploymentIdentity.properties.principalId diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs index 6878fa24c0..224254e7e2 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/SubscriptionTriggerer.cs @@ -47,10 +47,11 @@ private async Task> GetSubscriptionsToTrigger( var enabledSubscriptionsWithTargetFrequency = (await _context.Subscriptions .Where(s => s.Enabled) + .ToListAsync()) // TODO (https://github.com/dotnet/arcade-services/issues/3880) - Remove subscriptionIdGenerator .Where(s => _subscriptionIdGenerator.ShouldTriggerSubscription(s.Id)) - .ToListAsync()) - .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency); + .Where(s => s.PolicyObject?.UpdateFrequency == targetUpdateFrequency) + .ToList(); var workItemProducer = _workItemProducerFactory.CreateProducer(); diff --git a/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs b/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs index 662088d34f..b540cc0b12 100644 --- a/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs +++ b/test/ProductConstructionService.Api.Tests/ApiTestConfiguration.cs @@ -17,8 +17,6 @@ public static WebApplicationBuilder CreateTestHostBuilder() builder.Configuration["VmrPath"] = "vmrPath"; builder.Configuration["TmpPath"] = "tmpPath"; builder.Configuration["VmrUri"] = "https://vmr.com/uri"; - builder.Configuration["github-oauth-id"] = "clientId"; - builder.Configuration["github-oauth-secret"] = "clientSecret"; builder.Configuration["BuildAssetRegistrySqlConnectionString"] = "connectionString"; builder.Configuration["DataProtection:DataProtectionKeyUri"] = "https://keyvault.azure.com/secret/key"; builder.Configuration["DataProtection:KeyBlobUri"] = "https://blobs.azure.com/secret/key"; From 0974d8341db86141d40d1a690711c51b7c428f8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Fri, 30 Aug 2024 14:37:17 +0200 Subject: [PATCH 23/33] Disable invariant globalization in PCS jobs (#3905) --- .../ProductConstructionService.FeedCleaner.csproj | 2 ++ .../ProductConstructionService.LongestBuildPathUpdater.csproj | 2 ++ .../ProductConstructionService.SubscriptionTriggerer.csproj | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj index 018e1412c4..b137ad5e27 100644 --- a/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj +++ b/src/ProductConstructionService/ProductConstructionService.FeedCleaner/ProductConstructionService.FeedCleaner.csproj @@ -6,6 +6,8 @@ enable enable False + + false diff --git a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj index a80571b4b9..f936de02b0 100644 --- a/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj +++ b/src/ProductConstructionService/ProductConstructionService.LongestBuildPathUpdater/ProductConstructionService.LongestBuildPathUpdater.csproj @@ -6,6 +6,8 @@ enable enable False + + false diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj index 6421835cd9..1d2ecd1104 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/ProductConstructionService.SubscriptionTriggerer.csproj @@ -6,6 +6,8 @@ enable enable False + + false From fd78ad395872c14ad12a052825a32d1e38988475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 2 Sep 2024 15:02:37 +0200 Subject: [PATCH 24/33] Fix PCS->GitHub app based auth (#3907) --- .../DarcLib/GitHubClient.cs | 146 +++++++++--------- .../DarcLib/IRemoteGitRepo.cs | 10 +- .../appsettings.json | 3 +- .../GitHubClientTests.cs | 19 ++- 4 files changed, 94 insertions(+), 84 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 36c3118e2f..727a9ad51d 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -23,6 +23,7 @@ using Maestro.Common; using Microsoft.DotNet.DarcLib.Helpers; +#nullable enable namespace Microsoft.DotNet.DarcLib; public class GitHubClient : RemoteRepoBase, IRemoteGitRepo @@ -39,16 +40,16 @@ public class GitHubClient : RemoteRepoBase, IRemoteGitRepo private static readonly Regex PullRequestUriPattern = new(@"^/repos/(?[^/]+)/(?[^/]+)/pulls/(?\d+)$"); - private readonly Lazy _lazyClient; private readonly IRemoteTokenProvider _tokenProvider; private readonly ILogger _logger; private readonly JsonSerializerSettings _serializerSettings; private readonly string _userAgent = $"DarcLib-{DarcLibVersion}"; + private IGitHubClient? _lazyClient = null; static GitHubClient() { string version = Assembly.GetExecutingAssembly() - .GetCustomAttribute() + .GetCustomAttribute()! .InformationalVersion; _product = new ProductHeaderValue("DarcLib", version); } @@ -57,7 +58,7 @@ public GitHubClient( IRemoteTokenProvider remoteTokenProvider, IProcessManager processManager, ILogger logger, - IMemoryCache cache) + IMemoryCache? cache) : this(remoteTokenProvider, processManager, logger, null, cache) { } @@ -66,8 +67,8 @@ public GitHubClient( IRemoteTokenProvider remoteTokenProvider, IProcessManager processManager, ILogger logger, - string temporaryRepositoryPath, - IMemoryCache cache) + string? temporaryRepositoryPath, + IMemoryCache? cache) : base(remoteTokenProvider, processManager, temporaryRepositoryPath, cache, logger) { _tokenProvider = remoteTokenProvider; @@ -77,11 +78,8 @@ public GitHubClient( ContractResolver = new CamelCasePropertyNamesContractResolver(), NullValueHandling = NullValueHandling.Ignore }; - _lazyClient = new Lazy(CreateGitHubClient); } - public virtual IGitHubClient Client => _lazyClient.Value; - public bool AllowRetries { get; set; } = true; /// @@ -123,7 +121,7 @@ private async Task GetFileContentsAsync(string owner, string repo, strin responseContent = JObject.Parse(await response.Content.ReadAsStringAsync()); } - var content = responseContent["content"].ToString(); + var content = responseContent["content"]!.ToString(); _logger.LogInformation( $"Getting the contents of file '{filePath}' from repo '{owner}/{repo}' in branch '{branch}' succeeded!"); @@ -149,7 +147,7 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas $"Verifying if '{newBranch}' branch exists in repo '{repoUri}'. If not, we'll create it..."); (string owner, string repo) = ParseRepoUri(repoUri); - string latestSha = await GetLastCommitShaAsync(owner, repo, baseBranch); + string? latestSha = await GetLastCommitShaAsync(owner, repo, baseBranch); string body; var gitRef = $"refs/heads/{newBranch}"; @@ -225,7 +223,7 @@ public async Task DoesBranchExistAsync(string repoUri, string branch) (string owner, string repo) = ParseRepoUri(repoUri); try { - await Client.Repository.Branch.Get(owner, repo, branch); + await GetClient(repoUri).Repository.Branch.Get(owner, repo, branch); return true; } catch (NotFoundException) @@ -243,7 +241,7 @@ public async Task DoesBranchExistAsync(string repoUri, string branch) /// private async Task DeleteBranchAsync(string owner, string repo, string branch) { - await Client.Git.Reference.Delete(owner, repo, $"heads/{branch}"); + await GetClient(owner, repo).Git.Reference.Delete(owner, repo, $"heads/{branch}"); } /// @@ -259,8 +257,8 @@ public async Task> SearchPullRequestsAsync( string repoUri, string pullRequestBranch, PrStatus status, - string keyword = null, - string author = null) + string? keyword = null, + string? author = null) { (string owner, string repo) = ParseRepoUri(repoUri); var query = new StringBuilder(); @@ -288,9 +286,9 @@ public async Task> SearchPullRequestsAsync( responseContent = JObject.Parse(await response.Content.ReadAsStringAsync()); } - var items = JArray.Parse(responseContent["items"].ToString()); + var items = JArray.Parse(responseContent["items"]!.ToString()); - IEnumerable prs = items.Select(r => r["number"].ToObject()); + IEnumerable prs = items.Select(r => r["number"]!.ToObject()); return prs; } @@ -314,7 +312,7 @@ public async Task GetPullRequestStatusAsync(string pullRequestUrl) responseContent = JObject.Parse(await response.Content.ReadAsStringAsync()); } - if (Enum.TryParse(responseContent["state"].ToString(), true, out PrStatus status)) + if (Enum.TryParse(responseContent["state"]!.ToString(), true, out PrStatus status)) { if (status == PrStatus.Open) { @@ -323,7 +321,7 @@ public async Task GetPullRequestStatusAsync(string pullRequestUrl) if (status == PrStatus.Closed) { - if (bool.TryParse(responseContent["merged"].ToString(), out bool merged)) + if (bool.TryParse(responseContent["merged"]!.ToString(), out bool merged)) { if (merged) { @@ -346,7 +344,7 @@ public async Task GetPullRequestStatusAsync(string pullRequestUrl) public async Task GetPullRequestAsync(string pullRequestUrl) { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - Octokit.PullRequest pr = await Client.PullRequest.Get(owner, repo, id); + Octokit.PullRequest pr = await GetClient(owner, repo).PullRequest.Get(owner, repo, id); return new PullRequest { Title = pr.Title, @@ -370,7 +368,7 @@ public async Task CreatePullRequestAsync(string repoUri, PullRequest pul { Body = pullRequest.Description }; - Octokit.PullRequest createdPullRequest = await Client.PullRequest.Create(owner, repo, pr); + Octokit.PullRequest createdPullRequest = await GetClient(repoUri).PullRequest.Create(owner, repo, pr); return createdPullRequest.Url; } @@ -385,7 +383,7 @@ public async Task UpdatePullRequestAsync(string pullRequestUri, PullRequest pull { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUri); - await Client.PullRequest.Update( + await GetClient(owner, repo).PullRequest.Update( owner, repo, id, @@ -405,7 +403,7 @@ public async Task> GetPullRequestCommitsAsync(string pullRequestUr { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - IReadOnlyList pullRequestCommits = await Client.PullRequest.Commits(owner, repo, id); + IReadOnlyList pullRequestCommits = await GetClient(owner, repo).PullRequest.Commits(owner, repo, id); IList commits = new List(pullRequestCommits.Count); foreach (var commit in pullRequestCommits) @@ -427,7 +425,7 @@ public async Task> GetPullRequestCommitsAsync(string pullRequestUr public async Task MergeDependencyPullRequestAsync(string pullRequestUrl, MergePullRequestParameters parameters, string mergeCommitMessage) { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - Octokit.PullRequest pr = await Client.PullRequest.Get(owner, repo, id); + Octokit.PullRequest pr = await GetClient(owner, repo).PullRequest.Get(owner, repo, id); var mergePullRequest = new MergePullRequest { @@ -436,11 +434,11 @@ public async Task MergeDependencyPullRequestAsync(string pullRequestUrl, MergePu MergeMethod = parameters.SquashMerge ? PullRequestMergeMethod.Squash : PullRequestMergeMethod.Merge }; - await Client.PullRequest.Merge(owner, repo, id, mergePullRequest); + await GetClient(owner, repo).PullRequest.Merge(owner, repo, id, mergePullRequest); if (parameters.DeleteSourceBranch) { - await Client.Git.Reference.Delete(owner, repo, $"heads/{pr.Head.Ref}"); + await GetClient(owner, repo).Git.Reference.Delete(owner, repo, $"heads/{pr.Head.Ref}"); } } @@ -453,14 +451,14 @@ public async Task MergeDependencyPullRequestAsync(string pullRequestUrl, MergePu public async Task CreateOrUpdatePullRequestCommentAsync(string pullRequestUrl, string message) { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - IssueComment lastComment = (await Client.Issue.Comment.GetAllForIssue(owner, repo, id))[^1]; + IssueComment lastComment = (await GetClient(owner, repo).Issue.Comment.GetAllForIssue(owner, repo, id))[^1]; if (lastComment != null && lastComment.Body.EndsWith(CommentMarker)) { - await Client.Issue.Comment.Update(owner, repo, lastComment.Id, message + CommentMarker); + await GetClient(owner, repo).Issue.Comment.Update(owner, repo, lastComment.Id, message + CommentMarker); } else { - await Client.Issue.Comment.Create(owner, repo, id, message + CommentMarker); + await GetClient(owner, repo).Issue.Comment.Create(owner, repo, id, message + CommentMarker); } } @@ -477,13 +475,14 @@ private static string CheckRunId(MergePolicyEvaluationResult result, string sha) public async Task CreateOrUpdatePullRequestMergeStatusInfoAsync(string pullRequestUrl, IReadOnlyList evaluations) { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); + var client = GetClient(owner, repo); // Get the sha of the latest commit for the current PR - string prSha = (await Client.PullRequest.Get(owner, repo, id))?.Head?.Sha + string prSha = (await client.PullRequest.Get(owner, repo, id))?.Head?.Sha ?? throw new InvalidOperationException("We cannot find the sha of the pull request"); // Get a list of all the merge policies checks runs for the current PR List existingChecksRuns = - (await Client.Check.Run.GetAllForReference(owner, repo, prSha)) + (await client.Check.Run.GetAllForReference(owner, repo, prSha)) .CheckRuns.Where(e => e.ExternalId.StartsWith(MergePolicyConstants.MaestroMergePolicyCheckRunPrefix)).ToList(); var toBeAdded = evaluations.Where(e => existingChecksRuns.All(c => c.ExternalId != CheckRunId(e, prSha))); @@ -492,17 +491,17 @@ public async Task CreateOrUpdatePullRequestMergeStatusInfoAsync(string pullReque foreach (var newCheckRunValidation in toBeAdded) { - await Client.Check.Run.Create(owner, repo, CheckRunForAdd(newCheckRunValidation, prSha)); + await client.Check.Run.Create(owner, repo, CheckRunForAdd(newCheckRunValidation, prSha)); } foreach (var updatedCheckRun in toBeUpdated) { MergePolicyEvaluationResult eval = evaluations.Last(e => updatedCheckRun.ExternalId == CheckRunId(e, prSha)); CheckRunUpdate newCheckRunUpdateValidation = CheckRunForUpdate(eval); - await Client.Check.Run.Update(owner, repo, updatedCheckRun.Id, newCheckRunUpdateValidation); + await client.Check.Run.Update(owner, repo, updatedCheckRun.Id, newCheckRunUpdateValidation); } foreach (var deletedCheckRun in toBeDeleted) { - await Client.Check.Run.Update(owner, repo, deletedCheckRun.Id, CheckRunForDelete(deletedCheckRun)); + await client.Check.Run.Update(owner, repo, deletedCheckRun.Id, CheckRunForDelete(deletedCheckRun)); } } @@ -636,14 +635,14 @@ public async Task> GetFilesAtCommitAsync(string repoUri, string co TreeResponse recursiveTree = await GetRecursiveTreeAsync(owner, repo, pathTree.Sha); - GitFile[] files = await Task.WhenAll( + GitFile?[] files = await Task.WhenAll( recursiveTree.Tree.Where(treeItem => treeItem.Type == TreeType.Blob) .Select( async treeItem => { return await GetGitTreeItem(path, treeItem, owner, repo); })); - return [.. files]; + return [.. files.Where(f => f != null)]; } /// @@ -654,7 +653,7 @@ public async Task> GetFilesAtCommitAsync(string repoUri, string co /// Organization /// Repository /// Git file with tree item contents. - public async Task GetGitTreeItem(string path, TreeItem treeItem, string owner, string repo) + public async Task GetGitTreeItem(string path, TreeItem treeItem, string owner, string repo) { // If we have a cache available here, attempt to get the value in the cache // before making the request. Generally, we are requesting the same files for each @@ -707,7 +706,7 @@ private async Task GetGitItemImpl(string path, TreeItem treeItem, strin { try { - blob = await Client.Git.Blob.Get(owner, repo, treeItem.Sha); + blob = await GetClient(owner, repo).Git.Blob.Get(owner, repo, treeItem.Sha); break; } catch (Exception e) when ((e is ForbiddenException || e is AbuseException ) && attempts < maxAttempts) @@ -759,8 +758,8 @@ private async Task ExecuteRemoteGitCommandAsync( string repoUri, string requestUri, ILogger logger, - string body = null, - string versionOverride = null, + string? body = null, + string? versionOverride = null, int retryCount = 15, bool logFailure = true) { @@ -818,7 +817,7 @@ private HttpClient CreateHttpClient(string repoUri) /// Path to file /// Branch /// Sha of file or null if the file does not exist. - public async Task CheckIfFileExistsAsync(string repoUri, string filePath, string branch) + public async Task CheckIfFileExistsAsync(string repoUri, string filePath, string branch) { string commit; (string owner, string repo) = ParseRepoUri(repoUri); @@ -835,7 +834,7 @@ public async Task CheckIfFileExistsAsync(string repoUri, string filePath { content = JObject.Parse(await response.Content.ReadAsStringAsync()); } - commit = content["sha"].ToString(); + commit = content["sha"]!.ToString(); return commit; } @@ -851,7 +850,7 @@ public async Task CheckIfFileExistsAsync(string repoUri, string filePath /// Repository uri /// Branch to retrieve the latest sha for /// Latest sha. Nulls if no commits were found. - public Task GetLastCommitShaAsync(string repoUri, string branch) + public Task GetLastCommitShaAsync(string repoUri, string branch) { (string owner, string repo) = ParseRepoUri(repoUri); return GetLastCommitShaAsync(owner, repo, branch); @@ -863,7 +862,7 @@ public Task GetLastCommitShaAsync(string repoUri, string branch) /// Repository URI /// Sha of the commit /// Return the commit matching the specified sha. Null if no commit were found. - public Task GetCommitAsync(string repoUri, string sha) + public Task GetCommitAsync(string repoUri, string sha) { (string owner, string repo) = ParseRepoUri(repoUri); return GetCommitAsync(owner, repo, sha); @@ -876,10 +875,10 @@ public Task GetCommitAsync(string repoUri, string sha) /// Repository name /// Sha of the commit /// Return the commit matching the specified sha. Null if no commit were found. - private async Task GetCommitAsync(string owner, string repo, string sha) + private async Task GetCommitAsync(string owner, string repo, string sha) { - Repository repository = await Client.Repository.Get(owner, repo); - Octokit.GitHubCommit commit = await Client.Repository.Commit.Get(repository.Id, sha); + Repository repository = await GetClient(owner, repo).Repository.Get(owner, repo); + Octokit.GitHubCommit commit = await GetClient(owner, repo).Repository.Commit.Get(repository.Id, sha); if (commit == null) { return null; @@ -894,7 +893,7 @@ private async Task GetCommitAsync(string owner, string repo, string sha) /// Repository name /// Branch to retrieve the latest sha for /// Latest sha. Null if no commits were found. - private async Task GetLastCommitShaAsync(string owner, string repo, string branch) + private async Task GetLastCommitShaAsync(string owner, string repo, string branch) { try { @@ -908,7 +907,7 @@ private async Task GetLastCommitShaAsync(string owner, string repo, stri content = JObject.Parse(await response.Content.ReadAsStringAsync()); } - return content["sha"].ToString(); + return content["sha"]!.ToString(); } catch (HttpRequestException exc) when (exc.Message.Contains(((int)HttpStatusCode.NotFound).ToString()) || exc.Message.Contains(((int)HttpStatusCode.UnprocessableEntity).ToString())) @@ -926,7 +925,7 @@ public async Task> GetPullRequestChecksAsync(string pullRequestUrl) { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - var commits = await Client.Repository.PullRequest.Commits(owner, repo, id); + var commits = await GetClient(owner, repo).Repository.PullRequest.Commits(owner, repo, id); var lastCommitSha = commits[commits.Count - 1].Sha; return (await GetChecksFromStatusApiAsync(owner, repo, lastCommitSha)) @@ -946,7 +945,7 @@ public async Task> GetLatestPullRequestReviewsAsync(string pullReq { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); - var reviews = await Client.Repository.PullRequest.Review.GetAll(owner, repo, id); + var reviews = await GetClient(owner, repo).Repository.PullRequest.Review.GetAll(owner, repo, id); // Filter out comments because they could come after Approved/ChangedRequested, and they don't change the decision. reviews = reviews.Where(r => r.State != PullRequestReviewState.Commented).ToImmutableList(); @@ -982,7 +981,7 @@ private static ReviewState TranslateReviewState(PullRequestReviewState state) private async Task> GetChecksFromStatusApiAsync(string owner, string repo, string @ref) { - var status = await Client.Repository.Status.GetCombined(owner, repo, @ref); + var status = await GetClient(owner, repo).Repository.Status.GetCombined(owner, repo, @ref); return status.Statuses.Select( s => @@ -1004,7 +1003,7 @@ private async Task> GetChecksFromStatusApiAsync(string owner, strin private async Task> GetChecksFromChecksApiAsync(string owner, string repo, string @ref) { - var checkRuns = await Client.Check.Run.GetAllForReference(owner, repo, @ref); + var checkRuns = await GetClient(owner, repo).Check.Run.GetAllForReference(owner, repo, @ref); return checkRuns.CheckRuns.Select( run => { @@ -1027,9 +1026,21 @@ private async Task> GetChecksFromChecksApiAsync(string owner, strin .ToList(); } - private Octokit.GitHubClient CreateGitHubClient() + public virtual IGitHubClient GetClient(string repoUri) + { + _lazyClient ??= CreateGitHubClient(repoUri); + return _lazyClient; + } + + public virtual IGitHubClient GetClient(string owner, string repo) { - var token = _tokenProvider.GetTokenForRepository(GitHubApiUri); + _lazyClient ??= CreateGitHubClient($"https://github.com/{owner}/{repo}"); + return _lazyClient; + } + + private Octokit.GitHubClient CreateGitHubClient(string repoUri) + { + var token = _tokenProvider.GetTokenForRepository(repoUri); if (string.IsNullOrEmpty(token)) { throw new DarcException( @@ -1045,7 +1056,7 @@ private Octokit.GitHubClient CreateGitHubClient() private async Task GetRecursiveTreeAsync(string owner, string repo, string treeSha) { - TreeResponse tree = await Client.Git.Tree.GetRecursive(owner, repo, treeSha); + TreeResponse tree = await GetClient(owner, repo).Git.Tree.GetRecursive(owner, repo, treeSha); if (tree.Truncated) { throw new NotSupportedException( @@ -1059,13 +1070,13 @@ private async Task GetTreeForPathAsync(string owner, string repo, { var pathSegments = new Queue(path.Split('/', '\\')); var currentPath = new List(); - Octokit.Commit commit = await Client.Git.Commit.Get(owner, repo, commitSha); + Octokit.Commit commit = await GetClient(owner, repo).Git.Commit.Get(owner, repo, commitSha); string treeSha = commit.Tree.Sha; while (true) { - TreeResponse tree = await Client.Git.Tree.Get(owner, repo, treeSha); + TreeResponse tree = await GetClient(owner, repo).Git.Tree.Get(owner, repo, treeSha); if (tree.Truncated) { throw new NotSupportedException( @@ -1079,14 +1090,11 @@ private async Task GetTreeForPathAsync(string owner, string repo, string subfolder = pathSegments.Dequeue(); currentPath.Add(subfolder); - TreeItem subfolderItem = tree.Tree.Where(ti => ti.Type == TreeType.Tree) - .FirstOrDefault(ti => ti.Path == subfolder); - - if (subfolderItem == null) - { - throw new DirectoryNotFoundException( + TreeItem? subfolderItem = tree.Tree + .Where(ti => ti.Type == TreeType.Tree) + .FirstOrDefault(ti => ti.Path == subfolder) + ?? throw new DirectoryNotFoundException( $"The path '{string.Join("/", currentPath)}' could not be found."); - } treeSha = subfolderItem.Sha; } @@ -1109,7 +1117,7 @@ private async Task GetCommitMapForPathAsync( $"Getting the contents of file/files in '{path}' of repo '{repoUri}' at commit '{assetsProducedInCommit}'"); (string owner, string repo) = ParseRepoUri(repoUri); - List contents; + List? contents; using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, @@ -1120,7 +1128,7 @@ private async Task GetCommitMapForPathAsync( contents = JsonConvert.DeserializeObject>(await response.Content.ReadAsStringAsync()); } - foreach (GitHubContent content in contents) + foreach (GitHubContent content in contents!) { if (content.Type == GitHubContentType.File) { @@ -1196,7 +1204,7 @@ await CommitFilesAsync( branch, commitMessage, _logger, - await _tokenProvider.GetTokenForRepositoryAsync(GitHubApiUri), + await _tokenProvider.GetTokenForRepositoryAsync(repoUri), Constants.DarcBotName, Constants.DarcBotEmail); } @@ -1230,8 +1238,8 @@ public async Task GitDiffAsync(string repoUri, string baseVersion, stri { BaseVersion = baseVersion, TargetVersion = targetVersion, - Ahead = content["ahead_by"].Value(), - Behind = content["behind_by"].Value(), + Ahead = content["ahead_by"]!.Value(), + Behind = content["behind_by"]!.Value(), Valid = true }; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index b14a7b6e8c..62df62bcf2 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Threading.Tasks; +#nullable enable namespace Microsoft.DotNet.DarcLib; public interface IRemoteGitRepo : IGitRepoCloner, IGitRepo @@ -50,8 +51,8 @@ Task> SearchPullRequestsAsync( string repoUri, string pullRequestBranch, PrStatus status, - string keyword = null, - string author = null); + string? keyword = null, + string? author = null); /// /// Get the status of a pull request @@ -114,7 +115,7 @@ Task> SearchPullRequestsAsync( /// Repository uri /// Branch to retrieve the latest sha for /// Latest sha. Null if no commits were found. - Task GetLastCommitShaAsync(string repoUri, string branch); + Task GetLastCommitShaAsync(string repoUri, string branch); /// /// Get a commit in a repo @@ -122,7 +123,7 @@ Task> SearchPullRequestsAsync( /// Repository URI /// Sha of the commit /// Return the commit matching the specified sha. Null if no commit were found. - Task GetCommitAsync(string repoUri, string sha); + Task GetCommitAsync(string repoUri, string sha); /// /// Gets a list of file under a given path in a given revision. @@ -167,6 +168,7 @@ Task> SearchPullRequestsAsync( Task DoesBranchExistAsync(string repoUri, string branch); } +#nullable disable public class PullRequest { public string Title { get; set; } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json index 69b13a58f3..27c2d4aa8e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json @@ -4,7 +4,8 @@ "Default": "Information", "Microsoft.AspNetCore": "Warning", "Azure.Core": "Warning", - "Azure.Identity": "Warning" + "Azure.Identity": "Warning", + "Microsoft.IdentityModel.LoggingExtensions.IdentityLoggerAdapter": "Warning" } }, "AllowedHosts": "*", diff --git a/test/Microsoft.DotNet.DarcLib.Tests/GitHubClientTests.cs b/test/Microsoft.DotNet.DarcLib.Tests/GitHubClientTests.cs index 61ac574b8d..dfdfd646fc 100644 --- a/test/Microsoft.DotNet.DarcLib.Tests/GitHubClientTests.cs +++ b/test/Microsoft.DotNet.DarcLib.Tests/GitHubClientTests.cs @@ -140,13 +140,9 @@ public void SetGitHubClientObject(IGitHubClient value) _client = value; } - public override IGitHubClient Client - { - get - { - return _client; - } - } + public override IGitHubClient GetClient(string owner, string repo) => _client; + public override IGitHubClient GetClient(string repoUri) => _client; + public TestGitHubClient(string gitExecutable, string accessToken, ILogger logger, string temporaryRepositoryPath, IMemoryCache cache) : base(new ResolvedTokenProvider(accessToken), new ProcessManager(logger, gitExecutable), logger, temporaryRepositoryPath, cache) { @@ -225,7 +221,8 @@ public async Task TreeItemCacheTest(bool enableCache) octoKitGitMock.Setup(m => m.Blob).Returns(octoKitBlobClientMock.Object); octoKitClientMock.Setup(m => m.Git).Returns(octoKitGitMock.Object); - client.Setup(m => m.Client).Returns(octoKitClientMock.Object); + client.Setup(m => m.GetClient(It.IsAny())).Returns(octoKitClientMock.Object); + client.Setup(m => m.GetClient(It.IsAny(), It.IsAny())).Returns(octoKitClientMock.Object); // Request all but the last tree item in the list, then request the full set, then again. // For the cache scenario, we should have no cache hits on first pass, n-1 on the second, and N on the last @@ -311,7 +308,8 @@ public async Task GetGitTreeItemAbuseExceptionRetryTest() OctoKitGitBlobsClient.SetupSequence(m => m.Get(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(abuseException) .ReturnsAsync(blob); - client.Setup(m => m.Client).Returns(OctoKitGithubClient.Object); + client.Setup(m => m.GetClient(It.IsAny())).Returns(OctoKitGithubClient.Object); + client.Setup(m => m.GetClient(It.IsAny(), It.IsAny())).Returns(OctoKitGithubClient.Object); var resultGitFile = await client.Object.GetGitTreeItem(path, treeItem, owner, repo); resultGitFile.FilePath.Should().Be(path + "/" + treeItem.Path); @@ -336,7 +334,8 @@ public async Task GetGitTreeItemAbuseExceptionRetryWithRateLimitTest() OctoKitGitBlobsClient.SetupSequence(m => m.Get(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(abuseException) .ReturnsAsync(blob); - client.Setup(m => m.Client).Returns(OctoKitGithubClient.Object); + client.Setup(m => m.GetClient(It.IsAny())).Returns(OctoKitGithubClient.Object); + client.Setup(m => m.GetClient(It.IsAny(), It.IsAny())).Returns(OctoKitGithubClient.Object); var resultGitFile = await client.Object.GetGitTreeItem(path, treeItem, owner, repo); resultGitFile.FilePath.Should().Be(path + "/" + treeItem.Path); From 884b927cd3acb95ecbae107bf97f774439c71b70 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 15:37:45 +0200 Subject: [PATCH 25/33] [main] Update dependencies from dotnet/arcade (#3909) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 ++++++++++++------------ eng/Versions.props | 10 +++++----- global.json | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b32149d9a1..064556356b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c - + https://github.com/dotnet/arcade - 4460b755f3c7c89e9660d9580ff79afc4218dd85 + 80264e60280e2815e7d65871081ccac04a32445c https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 6aba592617..6b2f39315b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24421.4 - 8.0.0-beta.24421.4 - 8.0.0-beta.24421.4 - 8.0.0-beta.24421.4 - 8.0.0-beta.24421.4 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 + 8.0.0-beta.24426.2 17.4.1 1.1.0-beta.24376.1 1.1.0-beta.24376.1 diff --git a/global.json b/global.json index 278a08fc6d..6cb5cb8183 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24421.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24426.2" } } From 6bbdc124636c0ec76f60bfd4cb84642748c80ce8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 2 Sep 2024 16:21:36 +0200 Subject: [PATCH 26/33] Stop redirecting PCS INT to PROD (#3910) --- .../appsettings.Staging.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index 33f5f2c9dc..dee5c36424 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -28,9 +28,9 @@ "default": { "ManagedIdentityId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745" } - }, - "ApiRedirect": { - "Uri": "https://maestro.dot.net/", - "ManagedIdentityClientId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745" } + // "ApiRedirect": { + // "Uri": "https://maestro.dot.net/", + // "ManagedIdentityClientId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745" + // } } From 1ff3af562769c4722288413dae53072d7e3cb63f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Mon, 2 Sep 2024 16:34:34 +0200 Subject: [PATCH 27/33] Fix PCS client not sending the authorization header (#3911) --- .../ProductConstructionServiceApiOptions.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs index 11b43c1472..f4259736cb 100644 --- a/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.Client/ProductConstructionServiceApiOptions.cs @@ -3,6 +3,8 @@ using System; using System.Collections.Generic; +using Azure.Core.Pipeline; +using Azure.Core; using Maestro.Common.AppCredentials; namespace ProductConstructionService.Client @@ -67,5 +69,15 @@ public ProductConstructionServiceApiOptions(string accessToken, string managedId : this(PcsStagingUri, accessToken, managedIdentityId, disableInteractiveAuth) { } + + partial void InitializeOptions() + { + if (Credentials != null) + { + AddPolicy( + new BearerTokenAuthenticationPolicy(Credentials, Array.Empty()), + HttpPipelinePosition.PerCall); + } + } } } From a6b566db82983145bd598e84ce5db897c98b2693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Tue, 3 Sep 2024 13:42:20 +0200 Subject: [PATCH 28/33] Fix redis connection by turning on SSL (#3912) --- .../ProductConstructionService.Api/appsettings.Staging.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index dee5c36424..8dc3c18e2b 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -2,7 +2,7 @@ "KeyVaultName": "ProductConstructionInt", "ConnectionStrings": { "queues": "https://productconstructionint.queue.core.windows.net", - "redis": "prodconstaging.redis.cache.windows.net:6380" + "redis": "prodconstaging.redis.cache.windows.net:6380,ssl=true" }, "ManagedIdentityClientId": "1d43ba8a-c2a6-4fad-b064-6d8c16fc0745", "VmrUri": "https://github.com/maestro-auth-test/dnceng-vmr", From 6fb9d2bebe440ec450d11f090d47282fea1f5aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Tue, 3 Sep 2024 14:04:45 +0200 Subject: [PATCH 29/33] Fix VMR initialization and clone the URI from configuration (#3914) --- .../DarcLib/VirtualMonoRepo/VmrCloneManager.cs | 2 -- .../InitializationBackgroundService.cs | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCloneManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCloneManager.cs index 3c5a7a6e84..c5e1cf00d8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCloneManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCloneManager.cs @@ -75,8 +75,6 @@ public async Task PrepareVmrAsync( public async Task PrepareVmrAsync(string checkoutRef, CancellationToken cancellationToken) { - _vmrInfo.VmrUri = _vmrInfo.VmrUri; - ILocalGitRepo vmr = await PrepareVmrAsync( [_vmrInfo.VmrUri], [checkoutRef], diff --git a/src/ProductConstructionService/ProductConstructionService.Api/InitializationBackgroundService.cs b/src/ProductConstructionService/ProductConstructionService.Api/InitializationBackgroundService.cs index 5d523085b6..94fe1ad578 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/InitializationBackgroundService.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/InitializationBackgroundService.cs @@ -27,6 +27,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) // If Vmr cloning is taking more than 20 min, something is wrong var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken, new CancellationTokenSource(TimeSpan.FromMinutes(20)).Token); + IVmrInfo vmrInfo = scope.ServiceProvider.GetRequiredService(); + vmrInfo.VmrUri = options.VmrUri; IVmrCloneManager vmrCloneManager = scope.ServiceProvider.GetRequiredService(); await vmrCloneManager.PrepareVmrAsync("main", linkedTokenSource.Token); linkedTokenSource.Token.ThrowIfCancellationRequested(); From 9af3cafa76d8b51120d4b3db292509a6fb484b5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Tue, 3 Sep 2024 16:51:32 +0200 Subject: [PATCH 30/33] Silence noisy SQL logging in INT (#3915) --- .../PcsStartup.cs | 45 +++++++++---------- .../appsettings.Development.json | 5 --- .../appsettings.json | 5 ++- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 1ef3b05434..0542b150c8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -92,38 +92,37 @@ static PcsStartup() if (build == null) { - logger.LogError($"Could not find build with id {entity.BuildId} in BAR. Skipping dependency update."); + logger.LogError("Could not find build with id {buildId} in BAR. Skipping dependency update.", entity.BuildId); } else { bool hasAssetsWithPublishedLocations = build.Assets .Any(a => a.Locations.Any(al => al.Type != LocationType.None && !al.Location.EndsWith("/artifacts"))); - if (hasAssetsWithPublishedLocations) + if (!hasAssetsWithPublishedLocations) { - List subscriptionsToUpdate = context.Subscriptions - .Where(sub => - sub.Enabled && - sub.ChannelId == entity.ChannelId && - (sub.SourceRepository == entity.Build.GitHubRepository || sub.SourceDirectory == entity.Build.AzureDevOpsRepository) && - JsonExtensions.JsonValue(sub.PolicyString, "lax $.UpdateFrequency") == ((int)UpdateFrequency.EveryBuild).ToString()) - // TODO (https://github.com/dotnet/arcade-services/issues/3880) - .ToList() - .Where(sub => subscriptionIdGenerator.ShouldTriggerSubscription(sub.Id)) - .ToList(); - - foreach (Subscription subscription in subscriptionsToUpdate) - { - workItemProducer.ProduceWorkItemAsync(new() - { - BuildId = entity.BuildId, - SubscriptionId = subscription.Id - }).GetAwaiter().GetResult(); - } + logger.LogInformation("Skipping Dependency update for Build {buildId} because it contains no assets in valid locations", entity.BuildId); + return; } - else + + List subscriptionsToUpdate = context.Subscriptions + .Where(sub => + sub.Enabled && + sub.ChannelId == entity.ChannelId && + (sub.SourceRepository == entity.Build.GitHubRepository || sub.SourceDirectory == entity.Build.AzureDevOpsRepository) && + JsonExtensions.JsonValue(sub.PolicyString, "lax $.UpdateFrequency") == ((int)UpdateFrequency.EveryBuild).ToString()) + // TODO (https://github.com/dotnet/arcade-services/issues/3880) + .ToList() + .Where(sub => subscriptionIdGenerator.ShouldTriggerSubscription(sub.Id)) + .ToList(); + + foreach (Subscription subscription in subscriptionsToUpdate) { - logger.LogInformation($"Skipping Dependency update for Build {entity.BuildId} because it contains no assets in valid locations"); + workItemProducer.ProduceWorkItemAsync(new() + { + BuildId = entity.BuildId, + SubscriptionId = subscription.Id + }).GetAwaiter().GetResult(); } } }; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json index 13496043ed..6dda15a7e7 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json @@ -1,11 +1,6 @@ { "Logging": { "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning", - "Azure.Core": "Warning", - "Microsoft.EntityFrameworkCore.Database.Command": "Warning", - "Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware": "Information", "Microsoft.DotNet.DarcLib": "Trace" } }, diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json index 27c2d4aa8e..c0803ea3ac 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.json @@ -2,9 +2,12 @@ "Logging": { "LogLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning", + "Azure.Core": "Warning", "Azure.Identity": "Warning", + "Microsoft.AspNetCore": "Warning", + "Microsoft.AspNetCore.HttpLogging.HttpLoggingMiddleware": "Information", + "Microsoft.EntityFrameworkCore.Database.Command": "Warning", "Microsoft.IdentityModel.LoggingExtensions.IdentityLoggerAdapter": "Warning" } }, From 50d9e2ae3f516d03efd36425289da8154238537f Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 4 Sep 2024 09:41:33 +0200 Subject: [PATCH 31/33] Put the correct MI ID for Sub triggerer (#3918) --- .../appsettings.Staging.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json index 6e3c66bfe9..561855360e 100644 --- a/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.SubscriptionTriggerer/appsettings.Staging.json @@ -2,7 +2,7 @@ "ConnectionStrings": { "queues": "https://productconstructionint.queue.core.windows.net" }, - "ManagedIdentityClientId": "9729e72a-f381-4d59-a958-8aa94a18a8d2", + "ManagedIdentityClientId": "244cbfa2-f4d6-4048-ba28-b9334cbddfa7", "BuildAssetRegistrySqlConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False; User Id=USER_ID_PLACEHOLDER", "Kusto": { "Database": "engineeringdata", From a3a804b9e5f453058895f5cd9a0b7b71bced6f7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 4 Sep 2024 14:29:27 +0200 Subject: [PATCH 32/33] Do not log errors for expected cases (#3920) --- .../DarcLib/AzureDevOpsClient.cs | 3 +- .../DarcLib/GitHubClient.cs | 62 +++++++++---------- .../DarcLib/Helpers/HttpRequestManager.cs | 19 ++++-- .../PullRequestUpdater.cs | 19 +++--- 4 files changed, 59 insertions(+), 44 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index 0f928f5712..b894650c60 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -218,7 +218,8 @@ public async Task DoesBranchExistAsync(string repoUri, string branch) projectName, $"_apis/git/repositories/{repoName}/refs?filter=heads/{branch}", _logger, - versionOverride: "7.0"); + versionOverride: "7.0", + logFailure: false); var refs = ((JArray)content["value"]).ToObject>(); return refs.Any(refs => refs.Name == $"refs/heads/{branch}"); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 727a9ad51d..8e6f2c30fc 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -1,14 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; -using Octokit; using System; using System.Collections.Generic; +using System.Collections.Immutable; using System.IO; using System.Linq; using System.Net; @@ -17,11 +12,16 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; -using Microsoft.DotNet.Services.Utility; -using System.Collections.Immutable; -using Maestro.MergePolicyEvaluation; using Maestro.Common; +using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.Services.Utility; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; +using Octokit; #nullable enable namespace Microsoft.DotNet.DarcLib; @@ -128,7 +128,7 @@ private async Task GetFileContentsAsync(string owner, string repo, strin return this.GetDecodedContent(content); } - catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int) HttpStatusCode.NotFound).ToString())) + catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) { throw new DependencyFileNotFoundException(filePath, $"{owner}/{repo}", branch, reqEx); } @@ -140,11 +140,9 @@ private async Task GetFileContentsAsync(string owner, string repo, strin /// Repo to create a branch in /// New branch name /// Base of new branch - /// public async Task CreateBranchAsync(string repoUri, string newBranch, string baseBranch) { - _logger.LogInformation( - $"Verifying if '{newBranch}' branch exists in repo '{repoUri}'. If not, we'll create it..."); + _logger.LogInformation("Verifying if '{branch}' branch exists in repo '{repoUri}'. If not, we'll create it...", newBranch, repoUri); (string owner, string repo) = ParseRepoUri(repoUri); string? latestSha = await GetLastCommitShaAsync(owner, repo, baseBranch); @@ -162,7 +160,8 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/branches/{newBranch}", _logger, - retryCount: 0)) { } + retryCount: 0, + logFailure: false)) { } githubRef.Force = true; body = JsonConvert.SerializeObject(githubRef, _serializerSettings); @@ -173,11 +172,11 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas _logger, body)) { } - _logger.LogInformation($"Branch '{newBranch}' exists, updated"); + _logger.LogInformation("Branch '{branch}' exists, updated", newBranch); } - catch (HttpRequestException exc) when (exc.Message.Contains(((int) HttpStatusCode.NotFound).ToString())) + catch (HttpRequestException exc) when (exc.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) { - _logger.LogInformation($"'{newBranch}' branch doesn't exist. Creating it..."); + _logger.LogInformation("'{branch}' branch doesn't exist. Creating it...", newBranch); body = JsonConvert.SerializeObject(githubRef, _serializerSettings); using (await ExecuteRemoteGitCommandAsync( @@ -187,14 +186,14 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas _logger, body)) { } - _logger.LogInformation($"Branch '{newBranch}' created in repo '{repoUri}'!"); - + _logger.LogInformation("Branch '{branch}' created in repo '{repoUri}'", newBranch, repoUri); return; } catch (HttpRequestException exc) { _logger.LogError( - $"Checking if '{newBranch}' branch existed in repo '{repoUri}' failed with '{exc.Message}'"); + "Checking if '{branch}' branch existed in repo '{repoUri}' failed with '{error}'", + newBranch, repoUri, exc.Message); throw; } } @@ -426,7 +425,7 @@ public async Task MergeDependencyPullRequestAsync(string pullRequestUrl, MergePu { (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); Octokit.PullRequest pr = await GetClient(owner, repo).PullRequest.Get(owner, repo, id); - + var mergePullRequest = new MergePullRequest { CommitMessage = mergeCommitMessage, @@ -481,7 +480,7 @@ public async Task CreateOrUpdatePullRequestMergeStatusInfoAsync(string pullReque ?? throw new InvalidOperationException("We cannot find the sha of the pull request"); // Get a list of all the merge policies checks runs for the current PR - List existingChecksRuns = + List existingChecksRuns = (await client.Check.Run.GetAllForReference(owner, repo, prSha)) .CheckRuns.Where(e => e.ExternalId.StartsWith(MergePolicyConstants.MaestroMergePolicyCheckRunPrefix)).ToList(); @@ -494,7 +493,7 @@ public async Task CreateOrUpdatePullRequestMergeStatusInfoAsync(string pullReque await client.Check.Run.Create(owner, repo, CheckRunForAdd(newCheckRunValidation, prSha)); } foreach (var updatedCheckRun in toBeUpdated) - { + { MergePolicyEvaluationResult eval = evaluations.Last(e => updatedCheckRun.ExternalId == CheckRunId(e, prSha)); CheckRunUpdate newCheckRunUpdateValidation = CheckRunForUpdate(eval); await client.Check.Run.Update(owner, repo, updatedCheckRun.Id, newCheckRunUpdateValidation); @@ -709,7 +708,7 @@ private async Task GetGitItemImpl(string path, TreeItem treeItem, strin blob = await GetClient(owner, repo).Git.Blob.Get(owner, repo, treeItem.Sha); break; } - catch (Exception e) when ((e is ForbiddenException || e is AbuseException ) && attempts < maxAttempts) + catch (Exception e) when ((e is ForbiddenException || e is AbuseException) && attempts < maxAttempts) { // AbuseException exposes a retry-after field which lets us know how long we should wait. ForbiddenException does not, so use 60 seconds var retryAfterSeconds = 60; @@ -725,7 +724,7 @@ private async Task GetGitItemImpl(string path, TreeItem treeItem, strin } return blob; - + }, ex => _logger.LogError(ex, $"Failed to get blob at sha {treeItem.Sha}"), ex => ex is ApiException apiex && apiex.StatusCode >= HttpStatusCode.InternalServerError); @@ -830,7 +829,8 @@ private HttpClient CreateHttpClient(string repoUri) HttpMethod.Get, $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/contents/{filePath}?ref={branch}", - _logger)) + _logger, + logFailure: false)) { content = JObject.Parse(await response.Content.ReadAsStringAsync()); } @@ -838,7 +838,7 @@ private HttpClient CreateHttpClient(string repoUri) return commit; } - catch (HttpRequestException exc) when (exc.Message.Contains(((int) HttpStatusCode.NotFound).ToString())) + catch (HttpRequestException exc) when (exc.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) { return null; } @@ -954,8 +954,8 @@ public async Task> GetLatestPullRequestReviewsAsync(string pullReq var newestActionableReviews = reviews.GroupBy(r => r.User.Login) .ToDictionary(g => g.Key, g => (from r in reviews - where r.User.Login == g.Key - select r) + where r.User.Login == g.Key + select r) .OrderByDescending(r => r.SubmittedAt) .First()) .Values; @@ -982,7 +982,7 @@ private static ReviewState TranslateReviewState(PullRequestReviewState state) private async Task> GetChecksFromStatusApiAsync(string owner, string repo, string @ref) { var status = await GetClient(owner, repo).Repository.Status.GetCombined(owner, repo, @ref); - + return status.Statuses.Select( s => { @@ -1243,7 +1243,7 @@ public async Task GitDiffAsync(string repoUri, string baseVersion, stri Valid = true }; } - catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int) HttpStatusCode.NotFound).ToString())) + catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) { return GitDiff.UnknownDiff(); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/HttpRequestManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/HttpRequestManager.cs index bb7130e0b8..2bcde8f6d1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/HttpRequestManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/HttpRequestManager.cs @@ -96,8 +96,11 @@ public async Task ExecuteAsync(int retryCount = 3) errorDetails = $"Error details: {errorDetails}"; } - _logger.LogError($"A '{(int)response.StatusCode} - {response.StatusCode}' status was returned for a HTTP request. " + - $"We'll set the retries amount to 0. {errorDetails}"); + _logger.LogError( + "A '{httpCode} - {status}' status was returned for a HTTP request. We'll set the retries amount to 0. {error}", + (int)response.StatusCode, + response.StatusCode, + errorDetails); } retriesRemaining = 0; @@ -125,14 +128,22 @@ public async Task ExecuteAsync(int retryCount = 3) if (_logFailure) { _logger.LogError("There was an error executing method '{method}' against URI '{requestUri}' " + - "after {maxRetries} attempts. Exception: {exception}", _method, _requestUri, retryCount, ex); + "after {maxRetries} attempts. Exception: {exception}", + _method, + _requestUri, + retryCount, + ex); } throw; } else if (_logFailure) { _logger.LogWarning("There was an error executing method '{method}' against URI '{requestUri}'. " + - "{retriesRemaining} attempts remaining. Exception: {ex.ToString()}", _method, _requestUri, retriesRemaining, ex); + "{retriesRemaining} attempts remaining. Exception: {ex.ToString()}", + _method, + _requestUri, + retriesRemaining, + ex); } } --retriesRemaining; diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 486bc4fbc8..242e4f9488 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -215,8 +215,7 @@ await AddDependencyFlowEventsAsync( mergePolicyResult, prUrl); - await _pullRequestState.TryDeleteAsync(); - await _codeFlowState.TryDeleteAsync(); + await ClearAllStateAsync(); return PullRequestStatus.Completed; case MergePolicyCheckResult.FailedPolicies: @@ -253,8 +252,7 @@ await AddDependencyFlowEventsAsync( pr.MergePolicyResult, prUrl); - await _pullRequestState.TryDeleteAsync(); - await _codeFlowState.TryDeleteAsync(); + await ClearAllStateAsync(); // Also try to clean up the PR branch. try @@ -348,10 +346,7 @@ private async Task UpdateSubscriptionsForMergedPRAsync(IEnumerable From 5fab7b4b2cda8177d28df71f1d7096db45c825a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 4 Sep 2024 15:49:18 +0200 Subject: [PATCH 33/33] Improve dev UX when running PCS scenario tests + docs (#3919) --- .../appsettings.Development.json | 2 +- src/ProductConstructionService/Readme.md | 60 ++++++++++++++++--- .../MaestroScenarioTestBase.cs | 2 +- ...ctConstructionService.ScenarioTests.csproj | 2 +- .../TestParameters.cs | 48 ++++++++------- 5 files changed, 84 insertions(+), 30 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json index 6dda15a7e7..bc78ea7503 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json @@ -31,6 +31,6 @@ "Scope": [ "api://baf98f1b-374e-487d-af42-aa33807f11e4/Maestro.User" ] }, "ApiRedirect": { - "Uri": "https://maestro.int-dot.net/" + // "Uri": "https://maestro.int-dot.net/" } } diff --git a/src/ProductConstructionService/Readme.md b/src/ProductConstructionService/Readme.md index 8bf2403dbf..b85e65a5cc 100644 --- a/src/ProductConstructionService/Readme.md +++ b/src/ProductConstructionService/Readme.md @@ -1,10 +1,27 @@ -# Starting the service locally - -If you're running the service from VS, install the latest Preview VS. Be sure to install the `Azure Development => .NET Aspire SDK (Preview)` optional workload in the VS installer. - -If you're building the project using the command line, run `dotnet workload install aspire` or `dotnet workload update` to install/update the aspire workload. - -To run the Product Construction Service locally, set the `ProductConstructionService.AppHost` as Startup Project, and run with F5. +# Getting started with local development + +1. Install the latest Preview VS. + - Be sure to install the `Azure Development => .NET Aspire SDK (Preview)` optional workload in the VS installer. + - If you're building the project using the command line, run `dotnet workload install aspire` or `dotnet workload update` to install/update the aspire workload. +1. Install SQL Server Express: https://www.microsoft.com/en-us/sql-server/sql-server-downloads +1. Install Node.js LTS. When asked, at the end of installation, also opt-in for all necessary tools. +1. Install Entity Framework Core CLI by running `dotnet tool install --global dotnet-ef` +1. From the `src\Maestro\Maestro.Data` project directory, run `dotnet ef --msbuildprojectextensionspath database update`. + - Note that the generated files are in the root artifacts folder, not the artifacts folder within the Maestro.Data project folder +1. Join the `maestro-auth-test` org in GitHub (you will need to ask someone to manually add you to the org). +1. Make sure you can read the `ProductConstructionDev` keyvault. If you can't, ask someone to add you to the keyvault. +1. In SQL Server Object Explorer in Visual Studio, find the local SQLExpress database for the build asset registry and populate the Repositories table with the following rows: + + ```sql + INSERT INTO [Repositories] (RepositoryName, InstallationId) VALUES + ('https://github.com/maestro-auth-test/maestro-test', 289474), + ('https://github.com/maestro-auth-test/maestro-test2', 289474), + ('https://github.com/maestro-auth-test/maestro-test3', 289474), + ('https://github.com/maestro-auth-test/dnceng-vmr', 289474); + ``` +1. Install Docker Desktop: https://www.docker.com/products/docker-desktop + +# Configuring the service for local runs When running locally: - The service will attempt to read secrets from the [`ProductConstructionDev`](https://ms.portal.azure.com/#@microsoft.onmicrosoft.com/resource/subscriptions/cab65fc3-d077-467d-931f-3932eabf36d3/resourceGroups/product-construction-service/providers/Microsoft.KeyVault/vaults/ProductConstructionDev/overview) KeyVault, using your Microsoft credentials. If you're having some authentication double check the following: @@ -62,7 +79,36 @@ When running locally: dotnet build ``` +# Running the service locally + +To run the Product Construction Service locally: +1. Start Docker Desktop. +1. Set the `ProductConstructionService.AppHost` as Startup Project, and run with F5. + +# Running the Scenario Tests locally + +After you completed the steps to run the service locally, you can run the scenario tests against your local instance too: +- Right-click the `ProductConstructionService.ScenarioTests` project and select `Manage User Secrets` and add the following content: + ```json + { + "PCS_BASEURI": "https://localhost:53180/", + "GITHUB_TOKEN": "[FILL SAME TOKEN AS YOU WOULD FOR DARC]", + "DARC_PACKAGE_SOURCE": "[full path to your arcade-services]\\artifacts\\packages\\Debug\\NonShipping", + "DARC_VERSION": "0.0.99-dev" + } + ``` +- Build the Darc tool locally (it is run by the scenario tests): + ```ps + cd src\Microsoft.DotNet.Darc\Darc + dotnet pack -c Debug + ``` +- Open two Visual Studio instances. +- In the first instance, run the PCS service (instructions above). +- In the second instance, run any of the `ProductConstructionService.ScenarioTests` tests. +- After you have run the tests or the service locally, your local git credential manager might populate with the `dotnet-maestro-bot` account. You can log out of it by running `git credential-manager github logout dotnet-maestro-bot`. + # Instructions for recreating the Product Construction Service + Run the `provision.ps1` script by giving it the name of the subscription you want to create the service in. Note that keyvault and container registry names have to be unique on Azure, so you'll have to change these, or delete and purge the existing ones. This will create all of the necessary Azure resources. diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index d08bbc11ab..628cd785f9 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -53,7 +53,7 @@ public void SetTestParameters(TestParameters parameters) if (!string.IsNullOrEmpty(_parameters.MaestroToken)) { - _baseDarcRunArgs.AddRange(["--p", _parameters.MaestroToken]); + _baseDarcRunArgs.AddRange(["-p", _parameters.MaestroToken]); } } diff --git a/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj b/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj index 41833fa9c0..41cb7df177 100644 --- a/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj +++ b/test/ProductConstructionService.ScenarioTests/ProductConstructionService.ScenarioTests.csproj @@ -10,10 +10,10 @@ a523e3e9-b284-4c40-962d-e06de454891e + - diff --git a/test/ProductConstructionService.ScenarioTests/TestParameters.cs b/test/ProductConstructionService.ScenarioTests/TestParameters.cs index cb18c330ec..1464773b7e 100644 --- a/test/ProductConstructionService.ScenarioTests/TestParameters.cs +++ b/test/ProductConstructionService.ScenarioTests/TestParameters.cs @@ -3,6 +3,7 @@ using System.Reflection; using System.Runtime.InteropServices; +using Azure.Core; using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.DarcLib.Helpers; @@ -17,16 +18,16 @@ namespace ProductConstructionService.ScenarioTests; public class TestParameters : IDisposable { internal readonly TemporaryDirectory _dir; - private static readonly string pcsBaseUri; - private static readonly string? pcsToken; private static readonly string githubToken; private static readonly string darcPackageSource; - private static readonly string azdoToken; + private static readonly string? azdoToken; private static readonly bool isCI; private static readonly string? darcDir; private static readonly string? darcVersion; + private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; + static TestParameters() { IConfiguration userSecrets = new ConfigurationBuilder() @@ -36,17 +37,15 @@ static TestParameters() pcsBaseUri = Environment.GetEnvironmentVariable("PCS_BASEURI") ?? userSecrets["PCS_BASEURI"] ?? "https://product-construction-int.delightfuldune-c0f01ab0.westus2.azurecontainerapps.io/"; - pcsToken = Environment.GetEnvironmentVariable("PCS_TOKEN") ?? userSecrets["PCS_TOKEN"]; isCI = Environment.GetEnvironmentVariable("DARC_IS_CI")?.ToLower() == "true"; githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN") ?? userSecrets["GITHUB_TOKEN"] ?? throw new Exception("Please configure the GitHub token"); darcPackageSource = Environment.GetEnvironmentVariable("DARC_PACKAGE_SOURCE") ?? userSecrets["DARC_PACKAGE_SOURCE"] ?? throw new Exception("Please configure the Darc package source"); - azdoToken = Environment.GetEnvironmentVariable("AZDO_TOKEN") ?? userSecrets["AZDO_TOKEN"] - ?? throw new Exception("Please configure the Azure DevOps token"); + azdoToken = Environment.GetEnvironmentVariable("AZDO_TOKEN") + ?? userSecrets["AZDO_TOKEN"]; darcDir = Environment.GetEnvironmentVariable("DARC_DIR"); darcVersion = Environment.GetEnvironmentVariable("DARC_VERSION") ?? userSecrets["DARC_VERSION"]; - } /// If set to true, the test will attempt to use the non primary endpoint, if provided @@ -55,9 +54,9 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f var testDir = TemporaryDirectory.Get(); var testDirSharedWrapper = Shareable.Create(testDir); - IProductConstructionServiceApi pcsApi = pcsToken != null - ? PcsApiFactory.GetAuthenticated(pcsBaseUri, pcsToken, managedIdentityId: null, disableInteractiveAuth: isCI) - : PcsApiFactory.GetAnonymous(pcsBaseUri); + IProductConstructionServiceApi pcsApi = pcsBaseUri.Contains("localhost") + ? PcsApiFactory.GetAnonymous(pcsBaseUri) + : PcsApiFactory.GetAuthenticated(pcsBaseUri, accessToken: null, managedIdentityId: null, disableInteractiveAuth: isCI); var darcRootDir = darcDir; if (string.IsNullOrEmpty(darcRootDir)) @@ -68,11 +67,13 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f var darcExe = Path.Join(darcRootDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc"); var git = await TestHelpers.Which("git"); - var tokenProvider = AzureDevOpsTokenProvider.FromStaticOptions(new() + var azDoTokenProvider = AzureDevOpsTokenProvider.FromStaticOptions(new() { ["default"] = new() { - Token = azdoToken + Token = azdoToken, + UseLocalCredentials = !isCI, + DisableInteractiveAuth = isCI, } }); @@ -83,13 +84,22 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f new InMemoryCredentialStore(new Octokit.Credentials(githubToken))); var azDoClient = new AzureDevOpsClient( - tokenProvider, + azDoTokenProvider, new ProcessManager(new NUnitLogger(), git), new NUnitLogger(), testDirSharedWrapper.TryTake()!.Directory); return new TestParameters( - darcExe, git, pcsBaseUri, pcsToken, githubToken, pcsApi, githubApi, azDoClient, testDir, azdoToken, isCI); + darcExe, + git, + pcsBaseUri, + githubToken, + pcsApi, + githubApi, + azDoClient, + testDir, + azDoTokenProvider, + isCI); } private static async Task InstallDarc(IProductConstructionServiceApi pcsApi, Shareable toolPath) @@ -118,25 +128,23 @@ private TestParameters( string darcExePath, string gitExePath, string pcsBaseUri, - string? pcsToken, string gitHubToken, IProductConstructionServiceApi pcsApi, Octokit.GitHubClient gitHubApi, AzureDevOpsClient azdoClient, TemporaryDirectory dir, - string azdoToken, + IAzureDevOpsTokenProvider azdoTokenProvider, bool isCI) { _dir = dir; + _azdoTokenProvider = azdoTokenProvider; DarcExePath = darcExePath; GitExePath = gitExePath; MaestroBaseUri = pcsBaseUri; - MaestroToken = pcsToken; GitHubToken = gitHubToken; PcsApi = pcsApi; GitHubApi = gitHubApi; AzDoClient = azdoClient; - AzDoToken = azdoToken; IsCI = isCI; } @@ -150,7 +158,7 @@ private TestParameters( public string MaestroBaseUri { get; } - public string? MaestroToken { get; } + public string? MaestroToken => PcsApi.Options.Credentials?.GetToken(new TokenRequestContext(), default).Token; public string GitHubToken { get; } @@ -168,7 +176,7 @@ private TestParameters( public string AzureDevOpsProject { get; } = "internal"; - public string AzDoToken { get; } + public string AzDoToken => _azdoTokenProvider.GetTokenForAccount("default"); public bool IsCI { get; }