From 47e3672c762970073e4282bd563233da86bcca3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 17 Jul 2024 16:16:43 +0200 Subject: [PATCH] Add option to generate common CredScan suppression file for VMR sync (#3698) --- .../VirtualMonoRepo/InitializeOperation.cs | 1 + .../VirtualMonoRepo/UpdateOperation.cs | 1 + .../VmrSyncCommandLineOptions.cs | 3 + .../CredScanSuppressionsGenerator.cs | 209 ++++++++++++++++++ .../VirtualMonoRepo/IVmrInitializer.cs | 2 + .../DarcLib/VirtualMonoRepo/IVmrUpdater.cs | 2 + .../VirtualMonoRepo/VmrForwardFlower.cs | 3 + .../DarcLib/VirtualMonoRepo/VmrInfo.cs | 2 + .../DarcLib/VirtualMonoRepo/VmrInitializer.cs | 7 +- .../DarcLib/VirtualMonoRepo/VmrManagerBase.cs | 9 + .../VirtualMonoRepo/VmrRegistrations.cs | 1 + .../DarcLib/VirtualMonoRepo/VmrUpdater.cs | 10 +- .../VmrSyncRepoChangesTest.cs | 69 +++++- .../VmrTestsBase.cs | 14 +- 14 files changed, 321 insertions(+), 12 deletions(-) create mode 100644 src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs index c3ae7228ba..135fc8f0dd 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs @@ -38,6 +38,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs index 35d140b9d1..d1004fb698 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs @@ -36,6 +36,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs index e81609cfd9..0c2216109b 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs @@ -28,6 +28,9 @@ internal abstract class VmrSyncCommandLineOptions : VmrCommandLineOptions, IBase [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")] public bool GenerateCodeowners { get; set; } = false; + [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")] + public bool GenerateCredScanSuppressions { get; set; } = false; + [Option("discard-patches", Required = false, HelpText = "Delete .patch files created during the sync.")] public bool DiscardPatches { get; set; } = false; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs new file mode 100644 index 0000000000..8f8c86c768 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public interface ICredScanSuppressionsGenerator +{ + Task UpdateCredScanSuppressions(CancellationToken cancellationToken); +} + +public class CredScanSuppressionsGenerator : ICredScanSuppressionsGenerator +{ + private static readonly IReadOnlyCollection s_credScanSuppressionsLocations = new[] + { + new UnixPath(".config/" + VmrInfo.CredScanSuppressionsFileName), + new UnixPath("eng/" + VmrInfo.CredScanSuppressionsFileName), + }; + + private readonly IVmrInfo _vmrInfo; + private readonly ISourceManifest _sourceManifest; + private readonly ILocalGitClient _localGitClient; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _jsonOptions = new() + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public CredScanSuppressionsGenerator( + IVmrInfo vmrInfo, + ISourceManifest sourceManifest, + ILocalGitClient localGitClient, + IFileSystem fileSystem, + ILogger logger) + { + _vmrInfo = vmrInfo; + _sourceManifest = sourceManifest; + _localGitClient = localGitClient; + _fileSystem = fileSystem; + _logger = logger; + } + + /// + /// Generates the CredScanSuppressions.json file by gathering individual repo CredScanSuppressions.json files. + /// + public async Task UpdateCredScanSuppressions(CancellationToken cancellationToken) + { + _logger.LogInformation("Updating {credscansuppressions}...", VmrInfo.CredScanSuppressionsPath); + + var destPath = _vmrInfo.VmrPath / VmrInfo.CredScanSuppressionsPath; + + CredScanSuppressionFile vmrCredScanSuppressionsFile = new CredScanSuppressionFile(); + + _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(destPath) + ?? throw new Exception($"Failed to create {VmrInfo.CredScanSuppressionsFileName} in {destPath}")); + + bool fileExistedBefore = _fileSystem.FileExists(destPath); + + using (var destStream = _fileSystem.GetFileStream(destPath, FileMode.Create, FileAccess.Write)) + { + foreach (ISourceComponent component in _sourceManifest.Repositories.OrderBy(m => m.Path)) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, component.Path, cancellationToken); + + foreach (var submodule in _sourceManifest.Submodules.Where(s => s.Path.StartsWith($"{component.Path}/"))) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, submodule.Path, cancellationToken); + } + } + + JsonSerializer.Serialize(destStream, vmrCredScanSuppressionsFile, _jsonOptions); + } + + if (vmrCredScanSuppressionsFile.Suppressions.Count == 0) + { + _fileSystem.DeleteFile(destPath); + } + + bool fileExistsAfter = _fileSystem.FileExists(destPath); + + if (fileExistsAfter || fileExistedBefore) + { + await _localGitClient.StageAsync(_vmrInfo.VmrPath, new string[] { VmrInfo.CredScanSuppressionsPath }, cancellationToken); + } + + _logger.LogInformation("{credscansuppressions} updated", VmrInfo.CredScanSuppressionsPath); + } + + private async Task AddCredScanSuppressionsContent(CredScanSuppressionFile vmrCredScanSuppressionsFile, string repoPath, CancellationToken cancellationToken) + { + // CredScanSuppressions.json files are very restricted in size so we can safely work with them in-memory + var content = new List(); + + foreach (var location in s_credScanSuppressionsLocations) + { + cancellationToken.ThrowIfCancellationRequested(); + + var repoCredScanSuppressionsPath = _vmrInfo.VmrPath / VmrInfo.SourcesDir / repoPath / location; + + if (!_fileSystem.FileExists(repoCredScanSuppressionsPath)) continue; + + var repoCredScanSuppressionsFile = JsonSerializer.Deserialize(await _fileSystem.ReadAllTextAsync(repoCredScanSuppressionsPath), _jsonOptions); + + if (repoCredScanSuppressionsFile != null && repoCredScanSuppressionsFile.Suppressions != null) + { + foreach (var suppression in repoCredScanSuppressionsFile.Suppressions) + { + if (suppression.File != null) + { + for (int i = 0; i < suppression.File.Count; i++) + { + suppression.File[i] = FixCredScanSuppressionsRule(repoPath, suppression.File[i]); + } + } + } + + vmrCredScanSuppressionsFile.Suppressions.AddRange(repoCredScanSuppressionsFile.Suppressions); + } + } + } + + /// + /// Fixes a CredScanSuppressions.json file rule by prefixing the path with the VMR location and replacing backslash. + /// + private static string FixCredScanSuppressionsRule(string repoPath, string file) + { + if (string.IsNullOrWhiteSpace(file)) + { + return file; + } + + if (file.Contains('\\')) + { + file = file.Replace('\\', '/'); + } + + return $"/{VmrInfo.SourcesDir}/{repoPath}{(file.StartsWith('/') ? string.Empty : '/')}{file}"; + } +} + +class CredScanSuppressionFile +{ + [JsonPropertyName("tool")] + public string Tool { get; set; } = "Credential Scanner"; + [JsonPropertyName("suppressions")] + public List Suppressions { get; set; } = []; +} + +class CredScanSuppression +{ + [JsonPropertyName("_justification")] + public string Justification { get; set; } = ""; + [JsonPropertyName("placeholder")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? Placeholder { get; set; } + [JsonPropertyName("file")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? File { get; set; } +} + +class SingleStringOrArrayConverter : JsonConverter> +{ + public override List? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.Null: + return null; + case JsonTokenType.StartArray: + var list = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + var arrayItem = JsonSerializer.Deserialize(ref reader, options); + if (arrayItem != null) + { + list.Add(arrayItem); + } + } + return list; + default: + var item = JsonSerializer.Deserialize(ref reader, options); + return item != null ? [item] : null; + } + } + + public override void Write(Utf8JsonWriter writer, List objectToWrite, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, objectToWrite, objectToWrite.GetType(), options); + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs index 86acd1cdfb..d79a3710fc 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs @@ -23,6 +23,7 @@ public interface IVmrInitializer /// Path to VMR's README.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used Task InitializeRepository( string mappingName, @@ -34,6 +35,7 @@ Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs index 7f73283bf9..27bcd99faf 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs @@ -22,6 +22,7 @@ public interface IVmrUpdater /// Path to VMR's Component.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used /// True if the repository was updated, false if it was already up to date Task UpdateRepository( @@ -33,6 +34,7 @@ Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs index 2d1648791e..bbeafc5123 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs @@ -210,6 +210,7 @@ protected override async Task SameDirectionFlowAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: true, + generateCredScanSuppressions: true, discardPatches, cancellationToken); } @@ -255,6 +256,7 @@ await FlowCodeAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } @@ -330,6 +332,7 @@ .. submodules.Select(s => s.Path).Distinct().Select(VmrPatchHandler.GetExclusion componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs index 071c343544..1ac0391797 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs @@ -65,6 +65,7 @@ public class VmrInfo : IVmrInfo { public static readonly UnixPath SourcesDir = new("src"); public static readonly UnixPath CodeownersPath = new(".github/" + CodeownersFileName); + public static readonly UnixPath CredScanSuppressionsPath = new(".config/" + CredScanSuppressionsFileName); public const string SourceMappingsFileName = "source-mappings.json"; public const string GitInfoSourcesDir = "prereqs/git-info"; @@ -77,6 +78,7 @@ public class VmrInfo : IVmrInfo public const string ComponentListPath = "Components.md"; public const string ThirdPartyNoticesFileName = "THIRD-PARTY-NOTICES.txt"; public const string CodeownersFileName = "CODEOWNERS"; + public const string CredScanSuppressionsFileName = "CredScanSuppressions.json"; public static UnixPath RelativeSourcesDir { get; } = new("src"); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs index ef177c9cb3..a3394e292b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -55,6 +55,7 @@ public VmrInitializer( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -63,7 +64,7 @@ public VmrInitializer( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _vmrInfo = vmrInfo; _dependencyTracker = dependencyTracker; @@ -84,6 +85,7 @@ public async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -139,6 +141,7 @@ await InitializeRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -168,6 +171,7 @@ private async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -206,6 +210,7 @@ await UpdateRepoToRevisionAsync( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index cbdfc0104c..63d06f72d1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -30,6 +30,7 @@ public abstract class VmrManagerBase private readonly IThirdPartyNoticesGenerator _thirdPartyNoticesGenerator; private readonly IComponentListGenerator _componentListGenerator; private readonly ICodeownersGenerator _codeownersGenerator; + private readonly ICredScanSuppressionsGenerator _credScanSuppressionsGenerator; private readonly ILocalGitClient _localGitClient; private readonly IDependencyFileManager _dependencyFileManager; private readonly IFileSystem _fileSystem; @@ -46,6 +47,7 @@ protected VmrManagerBase( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator componentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -61,6 +63,7 @@ protected VmrManagerBase( _thirdPartyNoticesGenerator = thirdPartyNoticesGenerator; _componentListGenerator = componentListGenerator; _codeownersGenerator = codeownersGenerator; + _credScanSuppressionsGenerator = credScanSuppressionsGenerator; _localGitClient = localGitClient; _dependencyFileManager = dependencyFileManager; _fileSystem = fileSystem; @@ -79,6 +82,7 @@ public async Task> UpdateRepoToRevisionAs string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -141,6 +145,11 @@ public async Task> UpdateRepoToRevisionAs await _codeownersGenerator.UpdateCodeowners(cancellationToken); } + if (generateCredScanSuppressions) + { + await _credScanSuppressionsGenerator.UpdateCredScanSuppressions(cancellationToken); + } + // Commit without adding files as they were added to index directly await CommitAsync(commitMessage, author); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index fa4b0f1d86..cce0398af8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -63,6 +63,7 @@ public static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs index d33d305eb7..792969e0ae 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs @@ -67,6 +67,7 @@ public VmrUpdater( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -76,7 +77,7 @@ public VmrUpdater( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _logger = logger; _sourceManifest = sourceManifest; @@ -101,6 +102,7 @@ public async Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -132,6 +134,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -146,6 +149,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); return true; @@ -165,6 +169,7 @@ private async Task> UpdateRepositoryInter string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -244,6 +249,7 @@ await UpdateTargetVersionOnly( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -258,6 +264,7 @@ private async Task UpdateRepositoryRecursively( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -336,6 +343,7 @@ private async Task UpdateRepositoryRecursively( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs index e20a2d6f2b..b941979689 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs @@ -146,8 +146,10 @@ public async Task SubmodulesAreInlinedProperlyTest() await GitOperations.InitializeSubmodule(ProductRepoPath, submoduleName, SecondRepoPath, submoduleRelativePath); Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CodeownersPath, "# This is a first repo's CODEOWNERS\nfoo/bar @some/team"); + Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test"", ""file"": ""testfile"" } ] }"); await GitOperations.CommitAll(ProductRepoPath, "Add submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); var expectedFilesFromRepos = new List { @@ -155,6 +157,7 @@ public async Task SubmodulesAreInlinedProperlyTest() _dependencyRepoFilePath, submoduleFilePath, VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CodeownersPath, + VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CredScanSuppressionsPath, }; List expectedFiles = GetExpectedFilesInVmr( @@ -163,6 +166,7 @@ public async Task SubmodulesAreInlinedProperlyTest() expectedFilesFromRepos); expectedFiles.Add(VmrPath / VmrInfo.CodeownersPath); + expectedFiles.Add(VmrPath / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CompareFileContents(_productRepoFilePath, _productRepoFileName); @@ -179,6 +183,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -187,14 +207,17 @@ public async Task SubmodulesAreInlinedProperlyTest() await File.WriteAllTextAsync(SecondRepoPath / additionalFileName, "New external repo file"); Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CodeownersPath, "# This is a second repo's CODEOWNERS\n/xyz/foo @other/team"); + Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test2"", ""file"": ""testfile2"" } ] }"); await GitOperations.CommitAll(SecondRepoPath, "Adding new file in the submodule"); await GitOperations.PullMain(ProductRepoPath / submoduleRelativePath); await GitOperations.CommitAll(ProductRepoPath, "Checkout submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Add(additionalSubmoduleFilePath); expectedFiles.Add(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Add(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CheckFileContents( @@ -214,18 +237,42 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/externals/product-repo2/xyz/foo @other/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + }, + { + "_justification": "test2", + "file": [ + "/src/product-repo1/externals/product-repo2/testfile2" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); // Remove submodule await GitOperations.RemoveSubmodule(ProductRepoPath, submoduleRelativePath); await File.WriteAllTextAsync(VmrPath / VmrInfo.CodeownersPath, "My new content in the CODEOWNERS\n\n### CONTENT BELOW IS AUTO-GENERATED AND MANUAL CHANGES WILL BE OVERWRITTEN ###\n"); + await File.WriteAllTextAsync(VmrPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ ] }"); await GitOperations.CommitAll(ProductRepoPath, "Remove the submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Remove(submoduleFilePath); expectedFiles.Remove(additionalSubmoduleFilePath); expectedFiles.Remove(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Remove(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -243,6 +290,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); } protected override async Task CopyReposForCurrentTest() diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs index 8ecb3c196b..fe40bfdab9 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs @@ -160,29 +160,29 @@ protected async Task InitializeRepoAtLastCommit(string repoName, NativePath repo await CallDarcInitialize(repoName, commit, sourceMappings); } - protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false) + protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { var commit = await GitOperations.GetRepoLastCommit(repoPath); - await CallDarcUpdate(repoName, commit, generateCodeowners); + await CallDarcUpdate(repoName, commit, generateCodeowners, generateCredScanSuppressions); } private async Task CallDarcInitialize(string repository, string commit, LocalPath sourceMappingsPath) { using var scope = ServiceProvider.CreateScope(); var vmrInitializer = scope.ServiceProvider.GetRequiredService(); - await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, true, _cancellationToken.Token); + await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, false, true, _cancellationToken.Token); } - protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { - await CallDarcUpdate(repository, commit, [], generateCodeowners); + await CallDarcUpdate(repository, commit, [], generateCodeowners, generateCredScanSuppressions); } - protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { using var scope = ServiceProvider.CreateScope(); var vmrUpdater = scope.ServiceProvider.GetRequiredService(); - await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, true, _cancellationToken.Token); + await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, generateCredScanSuppressions, true, _cancellationToken.Token); } protected async Task CallDarcBackflow(string mappingName, NativePath repoPath, string branch, string? shaToFlow = null, int? buildToFlow = null)