Add option to generate common CredScan suppression file for VMR sync (#…

…3698)

src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs

@@ -38,6 +38,7 @@ protected override async Task ExecuteInternalAsync(
                 _options.ComponentTemplate,
                 _options.TpnTemplate,
                 _options.GenerateCodeowners,
+                _options.GenerateCredScanSuppressions,
                 _options.DiscardPatches,
                 cancellationToken);
 }

src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs

@@ -36,6 +36,7 @@ protected override async Task ExecuteInternalAsync(
                 _options.ComponentTemplate,
                 _options.TpnTemplate,
                 _options.GenerateCodeowners,
+                _options.GenerateCredScanSuppressions,
                 _options.DiscardPatches,
                 cancellationToken);
 }

src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs

@@ -28,6 +28,9 @@ internal abstract class VmrSyncCommandLineOptions : VmrCommandLineOptions, IBase
     [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")]
     public bool GenerateCodeowners { get; set; } = false;
 
+    [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")]
+    public bool GenerateCredScanSuppressions { get; set; } = false;
+
     [Option("discard-patches", Required = false, HelpText = "Delete .patch files created during the sync.")]
     public bool DiscardPatches { get; set; } = false;
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs

@@ -0,0 +1,209 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.DotNet.Darc.Models.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.Helpers;
+using Microsoft.Extensions.Logging;
+
+#nullable enable
+namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo;
+
+public interface ICredScanSuppressionsGenerator
+{
+    Task UpdateCredScanSuppressions(CancellationToken cancellationToken);
+}
+
+public class CredScanSuppressionsGenerator : ICredScanSuppressionsGenerator
+{
+    private static readonly IReadOnlyCollection<LocalPath> s_credScanSuppressionsLocations = new[]
+    {
+        new UnixPath(".config/" + VmrInfo.CredScanSuppressionsFileName),
+        new UnixPath("eng/" + VmrInfo.CredScanSuppressionsFileName),
+    };
+
+    private readonly IVmrInfo _vmrInfo;
+    private readonly ISourceManifest _sourceManifest;
+    private readonly ILocalGitClient _localGitClient;
+    private readonly IFileSystem _fileSystem;
+    private readonly ILogger<CredScanSuppressionsGenerator> _logger;
+    private readonly JsonSerializerOptions _jsonOptions = new()
+    {
+        AllowTrailingCommas = true,
+        ReadCommentHandling = JsonCommentHandling.Skip,
+        WriteIndented = true,
+        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+    };
+
+    public CredScanSuppressionsGenerator(
+        IVmrInfo vmrInfo,
+        ISourceManifest sourceManifest,
+        ILocalGitClient localGitClient,
+        IFileSystem fileSystem,
+        ILogger<CredScanSuppressionsGenerator> logger)
+    {
+        _vmrInfo = vmrInfo;
+        _sourceManifest = sourceManifest;
+        _localGitClient = localGitClient;
+        _fileSystem = fileSystem;
+        _logger = logger;
+    }
+
+    /// <summary>
+    /// Generates the CredScanSuppressions.json file by gathering individual repo CredScanSuppressions.json files.
+    /// </summary>
+    public async Task UpdateCredScanSuppressions(CancellationToken cancellationToken)
+    {
+        _logger.LogInformation("Updating {credscansuppressions}...", VmrInfo.CredScanSuppressionsPath);
+
+        var destPath = _vmrInfo.VmrPath / VmrInfo.CredScanSuppressionsPath;
+
+        CredScanSuppressionFile vmrCredScanSuppressionsFile = new CredScanSuppressionFile();
+
+        _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(destPath)
+            ?? throw new Exception($"Failed to create {VmrInfo.CredScanSuppressionsFileName} in {destPath}"));
+
+        bool fileExistedBefore = _fileSystem.FileExists(destPath);
+
+        using (var destStream = _fileSystem.GetFileStream(destPath, FileMode.Create, FileAccess.Write))
+        {
+            foreach (ISourceComponent component in _sourceManifest.Repositories.OrderBy(m => m.Path))
+            {
+                await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, component.Path, cancellationToken);
+
+                foreach (var submodule in _sourceManifest.Submodules.Where(s => s.Path.StartsWith($"{component.Path}/")))
+                {
+                    await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, submodule.Path, cancellationToken);
+                }
+            }
+
+            JsonSerializer.Serialize(destStream, vmrCredScanSuppressionsFile, _jsonOptions);
+        }
+
+        if (vmrCredScanSuppressionsFile.Suppressions.Count == 0)
+        {
+            _fileSystem.DeleteFile(destPath);
+        }
+
+        bool fileExistsAfter = _fileSystem.FileExists(destPath);
+
+        if (fileExistsAfter || fileExistedBefore)
+        {
+            await _localGitClient.StageAsync(_vmrInfo.VmrPath, new string[] { VmrInfo.CredScanSuppressionsPath }, cancellationToken);
+        }
+
+        _logger.LogInformation("{credscansuppressions} updated", VmrInfo.CredScanSuppressionsPath);
+    }
+
+    private async Task AddCredScanSuppressionsContent(CredScanSuppressionFile vmrCredScanSuppressionsFile, string repoPath, CancellationToken cancellationToken)
+    {
+        // CredScanSuppressions.json files are very restricted in size so we can safely work with them in-memory
+        var content = new List<string>();
+
+        foreach (var location in s_credScanSuppressionsLocations)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            var repoCredScanSuppressionsPath = _vmrInfo.VmrPath / VmrInfo.SourcesDir / repoPath / location;
+
+            if (!_fileSystem.FileExists(repoCredScanSuppressionsPath)) continue;
+
+            var repoCredScanSuppressionsFile = JsonSerializer.Deserialize<CredScanSuppressionFile>(await _fileSystem.ReadAllTextAsync(repoCredScanSuppressionsPath), _jsonOptions);
+
+            if (repoCredScanSuppressionsFile != null && repoCredScanSuppressionsFile.Suppressions != null)
+            {
+                foreach (var suppression in repoCredScanSuppressionsFile.Suppressions)
+                {
+                    if (suppression.File != null)
+                    {
+                        for (int i = 0; i < suppression.File.Count; i++)
+                        {
+                            suppression.File[i] = FixCredScanSuppressionsRule(repoPath, suppression.File[i]);
+                        }
+                    }
+                }
+
+                vmrCredScanSuppressionsFile.Suppressions.AddRange(repoCredScanSuppressionsFile.Suppressions);
+            }
+        }
+    }
+
+    /// <summary>
+    /// Fixes a CredScanSuppressions.json file rule by prefixing the path with the VMR location and replacing backslash.
+    /// </summary>
+    private static string FixCredScanSuppressionsRule(string repoPath, string file)
+    {
+        if (string.IsNullOrWhiteSpace(file))
+        {
+            return file;
+        }
+
+        if (file.Contains('\\'))
+        {
+            file = file.Replace('\\', '/');
+        }
+
+        return $"/{VmrInfo.SourcesDir}/{repoPath}{(file.StartsWith('/') ? string.Empty : '/')}{file}";
+    }
+}
+
+class CredScanSuppressionFile
+{
+    [JsonPropertyName("tool")]
+    public string Tool { get; set; } = "Credential Scanner";
+    [JsonPropertyName("suppressions")]
+    public List<CredScanSuppression> Suppressions { get; set; } = [];
+}
+
+class CredScanSuppression
+{
+    [JsonPropertyName("_justification")]
+    public string Justification { get; set; } = "";
+    [JsonPropertyName("placeholder")]
+    [JsonConverter(typeof(SingleStringOrArrayConverter))]
+    public List<string>? Placeholder { get; set; }
+    [JsonPropertyName("file")]
+    [JsonConverter(typeof(SingleStringOrArrayConverter))]
+    public List<string>? File { get; set; }
+}
+
+class SingleStringOrArrayConverter : JsonConverter<List<string>>
+{
+    public override List<string>? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        switch (reader.TokenType)
+        {
+            case JsonTokenType.Null:
+                return null;
+            case JsonTokenType.StartArray:
+                var list = new List<string>();
+                while (reader.Read())
+                {
+                    if (reader.TokenType == JsonTokenType.EndArray)
+                        break;
+                    var arrayItem = JsonSerializer.Deserialize<string>(ref reader, options);
+                    if (arrayItem != null)
+                    {
+                        list.Add(arrayItem);
+                    }
+                }
+                return list;
+            default:
+                var item = JsonSerializer.Deserialize<string>(ref reader, options);
+                return item != null ? [item] : null;
+        }
+    }
+
+    public override void Write(Utf8JsonWriter writer, List<string> objectToWrite, JsonSerializerOptions options)
+    {
+        JsonSerializer.Serialize(writer, objectToWrite, objectToWrite.GetType(), options);
+    }
+}

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs

@@ -23,6 +23,7 @@ public interface IVmrInitializer
     /// <param name="componentTemplatePath">Path to VMR's README.md template</param>
     /// <param name="tpnTemplatePath">Path to VMR's THIRD-PARTY-NOTICES.md template</param>
     /// <param name="generateCodeowners">Whether to generate a CODEOWNERS file</param>
+    /// <param name="generateCredScanSuppressions">Whether to generate a .config/CredScanSuppressions.json file</param>
     /// <param name="discardPatches">Whether to clean up genreated .patch files after their used</param>
     Task InitializeRepository(
         string mappingName,
@@ -34,6 +35,7 @@ Task InitializeRepository(
         string? componentTemplatePath,
         string? tpnTemplatePath,
         bool generateCodeowners,
+        bool generateCredScanSuppressions,
         bool discardPatches,
         CancellationToken cancellationToken);
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs

@@ -22,6 +22,7 @@ public interface IVmrUpdater
     /// <param name="componentTemplatePath">Path to VMR's Component.md template</param>
     /// <param name="tpnTemplatePath">Path to VMR's THIRD-PARTY-NOTICES.md template</param>
     /// <param name="generateCodeowners">Whether to generate a CODEOWNERS file</param>
+    /// <param name="generateCredScanSuppressions">Whether to generate a .config/CredScanSuppressions.json file</param>
     /// <param name="discardPatches">Whether to clean up genreated .patch files after their used</param>
     /// <returns>True if the repository was updated, false if it was already up to date</returns>
     Task<bool> UpdateRepository(
@@ -33,6 +34,7 @@ Task<bool> UpdateRepository(
         string? componentTemplatePath,
         string? tpnTemplatePath,
         bool generateCodeowners,
+        bool generateCredScanSuppressions,
         bool discardPatches,
         CancellationToken cancellationToken);
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs

@@ -210,6 +210,7 @@ protected override async Task<bool> SameDirectionFlowAsync(
                 componentTemplatePath: null,
                 tpnTemplatePath: null,
                 generateCodeowners: true,
+                generateCredScanSuppressions: true,
                 discardPatches,
                 cancellationToken);
         }
@@ -255,6 +256,7 @@ await FlowCodeAsync(
                 componentTemplatePath: null,
                 tpnTemplatePath: null,
                 generateCodeowners: false,
+                generateCredScanSuppressions: false,
                 discardPatches,
                 cancellationToken);
         }
@@ -330,6 +332,7 @@ .. submodules.Select(s => s.Path).Distinct().Select(VmrPatchHandler.GetExclusion
             componentTemplatePath: null,
             tpnTemplatePath: null,
             generateCodeowners: false,
+            generateCredScanSuppressions: false,
             discardPatches,
             cancellationToken);
     }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs

@@ -65,6 +65,7 @@ public class VmrInfo : IVmrInfo
 {
     public static readonly UnixPath SourcesDir = new("src");
     public static readonly UnixPath CodeownersPath = new(".github/" + CodeownersFileName);
+    public static readonly UnixPath CredScanSuppressionsPath = new(".config/" + CredScanSuppressionsFileName);
 
     public const string SourceMappingsFileName = "source-mappings.json";
     public const string GitInfoSourcesDir = "prereqs/git-info";
@@ -77,6 +78,7 @@ public class VmrInfo : IVmrInfo
     public const string ComponentListPath = "Components.md";
     public const string ThirdPartyNoticesFileName = "THIRD-PARTY-NOTICES.txt";
     public const string CodeownersFileName = "CODEOWNERS";
+    public const string CredScanSuppressionsFileName = "CredScanSuppressions.json";
 
     public static UnixPath RelativeSourcesDir { get; } = new("src");
 

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs

@@ -55,6 +55,7 @@ public VmrInitializer(
         IThirdPartyNoticesGenerator thirdPartyNoticesGenerator,
         IComponentListGenerator readmeComponentListGenerator,
         ICodeownersGenerator codeownersGenerator,
+        ICredScanSuppressionsGenerator credScanSuppressionsGenerator,
         ILocalGitClient localGitClient,
         ILocalGitRepoFactory localGitRepoFactory,
         IDependencyFileManager dependencyFileManager,
@@ -63,7 +64,7 @@ public VmrInitializer(
         ILogger<VmrUpdater> logger,
         ISourceManifest sourceManifest,
         IVmrInfo vmrInfo)
-        : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger)
+        : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger)
     {
         _vmrInfo = vmrInfo;
         _dependencyTracker = dependencyTracker;
@@ -84,6 +85,7 @@ public async Task InitializeRepository(
         string? componentTemplatePath,
         string? tpnTemplatePath,
         bool generateCodeowners,
+        bool generateCredScanSuppressions,
         bool discardPatches,
         CancellationToken cancellationToken)
     {
@@ -139,6 +141,7 @@ await InitializeRepository(
                     componentTemplatePath,
                     tpnTemplatePath,
                     generateCodeowners,
+                    generateCredScanSuppressions,
                     discardPatches,
                     cancellationToken);
             }
@@ -168,6 +171,7 @@ private async Task InitializeRepository(
         string? componentTemplatePath,
         string? tpnTemplatePath,
         bool generateCodeowners,
+        bool generateCredScanSuppressions,
         bool discardPatches,
         CancellationToken cancellationToken)
     {
@@ -206,6 +210,7 @@ await UpdateRepoToRevisionAsync(
             componentTemplatePath,
             tpnTemplatePath,
             generateCodeowners,
+            generateCredScanSuppressions,
             discardPatches,
             cancellationToken);