diff --git a/Directory.Packages.props b/Directory.Packages.props index 9b6391b015..8f7bd4cce9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -134,6 +134,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 27ef9330a7..79d4e01ed6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 954dbe9ea0..13dce6766b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 17.4.1 1.1.0-beta.24359.1 1.1.0-beta.24359.1 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index e1a1760022..238945cb5a 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -11,7 +11,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc "1.1.0-beta.24361.11" + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index d01739c128..ba3e7df815 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -140,11 +140,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9fd69fa7c9..57a41f0a3e 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -136,11 +136,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/eng/get-darc-version.ps1 b/eng/get-darc-version.ps1 deleted file mode 100644 index bfde0d76af..0000000000 --- a/eng/get-darc-version.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -param( - [Parameter(Mandatory=$True)] - [string]$maestroTestEndpoints, - [Parameter(Mandatory=$True)] - [string]$apiVersion -) - -$maestroTestEndpoint = $maestroTestEndpoints.Split(',')[0] -$versionEndpoint = "$maestroTestEndpoint/api/assets/darc-version?api-version=$apiVersion" -$latestDarcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content -Write-Host "##vso[task.setvariable variable=darcVersion]$latestDarcVersion" -Write-Host "Using Darc version $latestDarcVersion to run the tests" \ No newline at end of file diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index e34d8ee128..a15ec04f12 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -203,6 +203,27 @@ stages: .\darc\darc.exe get-default-channels --source-repo arcade-services --ci -t "$(GetAuthInfo.FederatedToken)" --bar-uri "$(GetAuthInfo.BarUri)" --debug displayName: Test Federated token authentication + - ${{ if in(variables['Build.SourceBranch'], 'refs/heads/main', 'refs/heads/production') }}: + - task: AzureCLI@2 + displayName: Test Darc add-build-to-channel + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $darcBuild = .\darc\darc.exe get-build ` + --repo "https://github.com/dotnet/arcade-services" ` + --commit "$(Build.SourceVersion)" ` + --ci ` + --output-format json | + ConvertFrom-Json + + .\darc\darc.exe add-build-to-channel ` + --id $darcBuild[0].id ` + --channel "General Testing" ` + --ci ` + --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) + - task: VSTest@2 displayName: Maestro Scenario Tests inputs: @@ -219,13 +240,3 @@ stages: DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts DARC_DIR: $(Build.SourcesDirectory)\darc DARC_IS_CI: true - - - powershell: | - nuget sources add -Name "arcade" -Source "https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json" - nuget sources add -Name "dotnet-core" -Source "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - displayName: Add nuget Sources - - - powershell: $(Build.SourcesDirectory)\eng\get-darc-version.ps1 - -maestroTestEndpoints "${{ parameters.MaestroTestEndpoints }}" - -apiVersion "2019-01-16" - displayName: Get DARC version diff --git a/global.json b/global.json index 4a8f4fcce4..4256a85777 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" } } diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs index 29aa0057fb..ec4b81b84a 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Maestro.Common; using Maestro.Common.AzureDevOpsTokens; using Microsoft.Arcade.Common; using Microsoft.DotNet.Darc.Helpers; @@ -76,6 +77,7 @@ protected Operation(ICommandLineOptions options, IServiceCollection? services = services.TryAddSingleton(s => s.GetRequiredService() ); + services.TryAddSingleton(_ => new RemoteTokenProvider(options.AzureDevOpsPat, options.GitHubPat)); Provider = services.BuildServiceProvider(); Logger = Provider.GetRequiredService>(); diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs index c3ae7228ba..135fc8f0dd 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs @@ -38,6 +38,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs index 35d140b9d1..d1004fb698 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs @@ -36,6 +36,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs index e81609cfd9..0c2216109b 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs @@ -28,6 +28,9 @@ internal abstract class VmrSyncCommandLineOptions : VmrCommandLineOptions, IBase [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")] public bool GenerateCodeowners { get; set; } = false; + [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")] + public bool GenerateCredScanSuppressions { get; set; } = false; + [Option("discard-patches", Required = false, HelpText = "Delete .patch files created during the sync.")] public bool DiscardPatches { get; set; } = false; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs new file mode 100644 index 0000000000..8f8c86c768 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public interface ICredScanSuppressionsGenerator +{ + Task UpdateCredScanSuppressions(CancellationToken cancellationToken); +} + +public class CredScanSuppressionsGenerator : ICredScanSuppressionsGenerator +{ + private static readonly IReadOnlyCollection s_credScanSuppressionsLocations = new[] + { + new UnixPath(".config/" + VmrInfo.CredScanSuppressionsFileName), + new UnixPath("eng/" + VmrInfo.CredScanSuppressionsFileName), + }; + + private readonly IVmrInfo _vmrInfo; + private readonly ISourceManifest _sourceManifest; + private readonly ILocalGitClient _localGitClient; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _jsonOptions = new() + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public CredScanSuppressionsGenerator( + IVmrInfo vmrInfo, + ISourceManifest sourceManifest, + ILocalGitClient localGitClient, + IFileSystem fileSystem, + ILogger logger) + { + _vmrInfo = vmrInfo; + _sourceManifest = sourceManifest; + _localGitClient = localGitClient; + _fileSystem = fileSystem; + _logger = logger; + } + + /// + /// Generates the CredScanSuppressions.json file by gathering individual repo CredScanSuppressions.json files. + /// + public async Task UpdateCredScanSuppressions(CancellationToken cancellationToken) + { + _logger.LogInformation("Updating {credscansuppressions}...", VmrInfo.CredScanSuppressionsPath); + + var destPath = _vmrInfo.VmrPath / VmrInfo.CredScanSuppressionsPath; + + CredScanSuppressionFile vmrCredScanSuppressionsFile = new CredScanSuppressionFile(); + + _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(destPath) + ?? throw new Exception($"Failed to create {VmrInfo.CredScanSuppressionsFileName} in {destPath}")); + + bool fileExistedBefore = _fileSystem.FileExists(destPath); + + using (var destStream = _fileSystem.GetFileStream(destPath, FileMode.Create, FileAccess.Write)) + { + foreach (ISourceComponent component in _sourceManifest.Repositories.OrderBy(m => m.Path)) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, component.Path, cancellationToken); + + foreach (var submodule in _sourceManifest.Submodules.Where(s => s.Path.StartsWith($"{component.Path}/"))) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, submodule.Path, cancellationToken); + } + } + + JsonSerializer.Serialize(destStream, vmrCredScanSuppressionsFile, _jsonOptions); + } + + if (vmrCredScanSuppressionsFile.Suppressions.Count == 0) + { + _fileSystem.DeleteFile(destPath); + } + + bool fileExistsAfter = _fileSystem.FileExists(destPath); + + if (fileExistsAfter || fileExistedBefore) + { + await _localGitClient.StageAsync(_vmrInfo.VmrPath, new string[] { VmrInfo.CredScanSuppressionsPath }, cancellationToken); + } + + _logger.LogInformation("{credscansuppressions} updated", VmrInfo.CredScanSuppressionsPath); + } + + private async Task AddCredScanSuppressionsContent(CredScanSuppressionFile vmrCredScanSuppressionsFile, string repoPath, CancellationToken cancellationToken) + { + // CredScanSuppressions.json files are very restricted in size so we can safely work with them in-memory + var content = new List(); + + foreach (var location in s_credScanSuppressionsLocations) + { + cancellationToken.ThrowIfCancellationRequested(); + + var repoCredScanSuppressionsPath = _vmrInfo.VmrPath / VmrInfo.SourcesDir / repoPath / location; + + if (!_fileSystem.FileExists(repoCredScanSuppressionsPath)) continue; + + var repoCredScanSuppressionsFile = JsonSerializer.Deserialize(await _fileSystem.ReadAllTextAsync(repoCredScanSuppressionsPath), _jsonOptions); + + if (repoCredScanSuppressionsFile != null && repoCredScanSuppressionsFile.Suppressions != null) + { + foreach (var suppression in repoCredScanSuppressionsFile.Suppressions) + { + if (suppression.File != null) + { + for (int i = 0; i < suppression.File.Count; i++) + { + suppression.File[i] = FixCredScanSuppressionsRule(repoPath, suppression.File[i]); + } + } + } + + vmrCredScanSuppressionsFile.Suppressions.AddRange(repoCredScanSuppressionsFile.Suppressions); + } + } + } + + /// + /// Fixes a CredScanSuppressions.json file rule by prefixing the path with the VMR location and replacing backslash. + /// + private static string FixCredScanSuppressionsRule(string repoPath, string file) + { + if (string.IsNullOrWhiteSpace(file)) + { + return file; + } + + if (file.Contains('\\')) + { + file = file.Replace('\\', '/'); + } + + return $"/{VmrInfo.SourcesDir}/{repoPath}{(file.StartsWith('/') ? string.Empty : '/')}{file}"; + } +} + +class CredScanSuppressionFile +{ + [JsonPropertyName("tool")] + public string Tool { get; set; } = "Credential Scanner"; + [JsonPropertyName("suppressions")] + public List Suppressions { get; set; } = []; +} + +class CredScanSuppression +{ + [JsonPropertyName("_justification")] + public string Justification { get; set; } = ""; + [JsonPropertyName("placeholder")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? Placeholder { get; set; } + [JsonPropertyName("file")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? File { get; set; } +} + +class SingleStringOrArrayConverter : JsonConverter> +{ + public override List? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.Null: + return null; + case JsonTokenType.StartArray: + var list = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + var arrayItem = JsonSerializer.Deserialize(ref reader, options); + if (arrayItem != null) + { + list.Add(arrayItem); + } + } + return list; + default: + var item = JsonSerializer.Deserialize(ref reader, options); + return item != null ? [item] : null; + } + } + + public override void Write(Utf8JsonWriter writer, List objectToWrite, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, objectToWrite, objectToWrite.GetType(), options); + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs index 86acd1cdfb..d79a3710fc 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs @@ -23,6 +23,7 @@ public interface IVmrInitializer /// Path to VMR's README.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used Task InitializeRepository( string mappingName, @@ -34,6 +35,7 @@ Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs index 7f73283bf9..27bcd99faf 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs @@ -22,6 +22,7 @@ public interface IVmrUpdater /// Path to VMR's Component.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used /// True if the repository was updated, false if it was already up to date Task UpdateRepository( @@ -33,6 +34,7 @@ Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs index 2d1648791e..bbeafc5123 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs @@ -210,6 +210,7 @@ protected override async Task SameDirectionFlowAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: true, + generateCredScanSuppressions: true, discardPatches, cancellationToken); } @@ -255,6 +256,7 @@ await FlowCodeAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } @@ -330,6 +332,7 @@ .. submodules.Select(s => s.Path).Distinct().Select(VmrPatchHandler.GetExclusion componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs index 071c343544..1ac0391797 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs @@ -65,6 +65,7 @@ public class VmrInfo : IVmrInfo { public static readonly UnixPath SourcesDir = new("src"); public static readonly UnixPath CodeownersPath = new(".github/" + CodeownersFileName); + public static readonly UnixPath CredScanSuppressionsPath = new(".config/" + CredScanSuppressionsFileName); public const string SourceMappingsFileName = "source-mappings.json"; public const string GitInfoSourcesDir = "prereqs/git-info"; @@ -77,6 +78,7 @@ public class VmrInfo : IVmrInfo public const string ComponentListPath = "Components.md"; public const string ThirdPartyNoticesFileName = "THIRD-PARTY-NOTICES.txt"; public const string CodeownersFileName = "CODEOWNERS"; + public const string CredScanSuppressionsFileName = "CredScanSuppressions.json"; public static UnixPath RelativeSourcesDir { get; } = new("src"); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs index ef177c9cb3..a3394e292b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -55,6 +55,7 @@ public VmrInitializer( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -63,7 +64,7 @@ public VmrInitializer( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _vmrInfo = vmrInfo; _dependencyTracker = dependencyTracker; @@ -84,6 +85,7 @@ public async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -139,6 +141,7 @@ await InitializeRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -168,6 +171,7 @@ private async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -206,6 +210,7 @@ await UpdateRepoToRevisionAsync( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index cbdfc0104c..63d06f72d1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -30,6 +30,7 @@ public abstract class VmrManagerBase private readonly IThirdPartyNoticesGenerator _thirdPartyNoticesGenerator; private readonly IComponentListGenerator _componentListGenerator; private readonly ICodeownersGenerator _codeownersGenerator; + private readonly ICredScanSuppressionsGenerator _credScanSuppressionsGenerator; private readonly ILocalGitClient _localGitClient; private readonly IDependencyFileManager _dependencyFileManager; private readonly IFileSystem _fileSystem; @@ -46,6 +47,7 @@ protected VmrManagerBase( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator componentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -61,6 +63,7 @@ protected VmrManagerBase( _thirdPartyNoticesGenerator = thirdPartyNoticesGenerator; _componentListGenerator = componentListGenerator; _codeownersGenerator = codeownersGenerator; + _credScanSuppressionsGenerator = credScanSuppressionsGenerator; _localGitClient = localGitClient; _dependencyFileManager = dependencyFileManager; _fileSystem = fileSystem; @@ -79,6 +82,7 @@ public async Task> UpdateRepoToRevisionAs string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -141,6 +145,11 @@ public async Task> UpdateRepoToRevisionAs await _codeownersGenerator.UpdateCodeowners(cancellationToken); } + if (generateCredScanSuppressions) + { + await _credScanSuppressionsGenerator.UpdateCredScanSuppressions(cancellationToken); + } + // Commit without adding files as they were added to index directly await CommitAsync(commitMessage, author); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index fa4b0f1d86..cce0398af8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -63,6 +63,7 @@ public static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs index d33d305eb7..792969e0ae 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs @@ -67,6 +67,7 @@ public VmrUpdater( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -76,7 +77,7 @@ public VmrUpdater( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _logger = logger; _sourceManifest = sourceManifest; @@ -101,6 +102,7 @@ public async Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -132,6 +134,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -146,6 +149,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); return true; @@ -165,6 +169,7 @@ private async Task> UpdateRepositoryInter string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -244,6 +249,7 @@ await UpdateTargetVersionOnly( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -258,6 +264,7 @@ private async Task UpdateRepositoryRecursively( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -336,6 +343,7 @@ private async Task UpdateRepositoryRecursively( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs index e20a2d6f2b..b941979689 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs @@ -146,8 +146,10 @@ public async Task SubmodulesAreInlinedProperlyTest() await GitOperations.InitializeSubmodule(ProductRepoPath, submoduleName, SecondRepoPath, submoduleRelativePath); Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CodeownersPath, "# This is a first repo's CODEOWNERS\nfoo/bar @some/team"); + Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test"", ""file"": ""testfile"" } ] }"); await GitOperations.CommitAll(ProductRepoPath, "Add submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); var expectedFilesFromRepos = new List { @@ -155,6 +157,7 @@ public async Task SubmodulesAreInlinedProperlyTest() _dependencyRepoFilePath, submoduleFilePath, VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CodeownersPath, + VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CredScanSuppressionsPath, }; List expectedFiles = GetExpectedFilesInVmr( @@ -163,6 +166,7 @@ public async Task SubmodulesAreInlinedProperlyTest() expectedFilesFromRepos); expectedFiles.Add(VmrPath / VmrInfo.CodeownersPath); + expectedFiles.Add(VmrPath / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CompareFileContents(_productRepoFilePath, _productRepoFileName); @@ -179,6 +183,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -187,14 +207,17 @@ public async Task SubmodulesAreInlinedProperlyTest() await File.WriteAllTextAsync(SecondRepoPath / additionalFileName, "New external repo file"); Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CodeownersPath, "# This is a second repo's CODEOWNERS\n/xyz/foo @other/team"); + Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test2"", ""file"": ""testfile2"" } ] }"); await GitOperations.CommitAll(SecondRepoPath, "Adding new file in the submodule"); await GitOperations.PullMain(ProductRepoPath / submoduleRelativePath); await GitOperations.CommitAll(ProductRepoPath, "Checkout submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Add(additionalSubmoduleFilePath); expectedFiles.Add(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Add(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CheckFileContents( @@ -214,18 +237,42 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/externals/product-repo2/xyz/foo @other/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + }, + { + "_justification": "test2", + "file": [ + "/src/product-repo1/externals/product-repo2/testfile2" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); // Remove submodule await GitOperations.RemoveSubmodule(ProductRepoPath, submoduleRelativePath); await File.WriteAllTextAsync(VmrPath / VmrInfo.CodeownersPath, "My new content in the CODEOWNERS\n\n### CONTENT BELOW IS AUTO-GENERATED AND MANUAL CHANGES WILL BE OVERWRITTEN ###\n"); + await File.WriteAllTextAsync(VmrPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ ] }"); await GitOperations.CommitAll(ProductRepoPath, "Remove the submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Remove(submoduleFilePath); expectedFiles.Remove(additionalSubmoduleFilePath); expectedFiles.Remove(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Remove(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -243,6 +290,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); } protected override async Task CopyReposForCurrentTest() diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs index 8ecb3c196b..fe40bfdab9 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs @@ -160,29 +160,29 @@ protected async Task InitializeRepoAtLastCommit(string repoName, NativePath repo await CallDarcInitialize(repoName, commit, sourceMappings); } - protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false) + protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { var commit = await GitOperations.GetRepoLastCommit(repoPath); - await CallDarcUpdate(repoName, commit, generateCodeowners); + await CallDarcUpdate(repoName, commit, generateCodeowners, generateCredScanSuppressions); } private async Task CallDarcInitialize(string repository, string commit, LocalPath sourceMappingsPath) { using var scope = ServiceProvider.CreateScope(); var vmrInitializer = scope.ServiceProvider.GetRequiredService(); - await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, true, _cancellationToken.Token); + await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, false, true, _cancellationToken.Token); } - protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { - await CallDarcUpdate(repository, commit, [], generateCodeowners); + await CallDarcUpdate(repository, commit, [], generateCodeowners, generateCredScanSuppressions); } - protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { using var scope = ServiceProvider.CreateScope(); var vmrUpdater = scope.ServiceProvider.GetRequiredService(); - await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, true, _cancellationToken.Token); + await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, generateCredScanSuppressions, true, _cancellationToken.Token); } protected async Task CallDarcBackflow(string mappingName, NativePath repoPath, string branch, string? shaToFlow = null, int? buildToFlow = null)