From 51436411c0c358de21f0f577806ee9088f57af62 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:21:28 +0200 Subject: [PATCH 01/14] [main] Update dependencies from dotnet/dnceng (#3704) Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 4 ++-- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index a3f4123928..f6abcedb6f 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dnceng.secretmanager": { - "version": "1.1.0-beta.24325.1", + "version": "1.1.0-beta.24351.3", "commands": [ "secret-manager" ] @@ -15,7 +15,7 @@ ] }, "microsoft.dnceng.configuration.bootstrap": { - "version": "1.1.0-beta.24325.1", + "version": "1.1.0-beta.24351.3", "commands": [ "bootstrap-dnceng-configuration" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d45cb5bb23..086e95370b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -115,11 +115,11 @@ https://github.com/dotnet/arcade 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/dnceng b6850078ffe074f4ad7d4952486d4997b1e029b6 - + https://github.com/dotnet/dnceng b6850078ffe074f4ad7d4952486d4997b1e029b6 diff --git a/eng/Versions.props b/eng/Versions.props index a60051b5ea..2e32a874df 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -37,8 +37,8 @@ 1.1.0-beta.24353.1 1.1.0-beta.24353.1 1.1.0-beta.24353.1 - 1.1.0-beta.24325.1 - 1.1.0-beta.24325.1 + 1.1.0-beta.24351.3 + 1.1.0-beta.24351.3 From 04e077a419fbd8b9d01d0d16467b324c5d9784c1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 10:22:03 +0200 Subject: [PATCH 02/14] [main] Update dependencies from dotnet/arcade (#3705) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 ++++++++++++------------ eng/Versions.props | 10 +++++----- global.json | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 086e95370b..f59d773f33 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 - + https://github.com/dotnet/arcade - 761c516b64fee3941d8909d24205ced835eed83e + 8b879da4e449c48d99f3f642fc429379a64e8fe8 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 2e32a874df..f8d27f6951 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24328.2 - 8.0.0-beta.24328.2 - 8.0.0-beta.24328.2 - 8.0.0-beta.24328.2 - 8.0.0-beta.24328.2 + 8.0.0-beta.24352.1 + 8.0.0-beta.24352.1 + 8.0.0-beta.24352.1 + 8.0.0-beta.24352.1 + 8.0.0-beta.24352.1 17.4.1 1.1.0-beta.24353.1 1.1.0-beta.24353.1 diff --git a/global.json b/global.json index 45aa063ffb..4a8f4fcce4 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24328.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1" } } From 8bb084a96325673fd97d143a3a3a1316aa557049 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:43:40 +0200 Subject: [PATCH 03/14] [main] Update dependencies from dotnet/dnceng-shared (#3709) Co-authored-by: dotnet-maestro[bot] Co-authored-by: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Co-authored-by: dkurepa --- Directory.Packages.props | 4 +- eng/Version.Details.xml | 88 ++++++++++++++++++++-------------------- eng/Versions.props | 44 ++++++++++---------- 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index db97a9b60a..9b6391b015 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -120,7 +120,7 @@ - + @@ -143,4 +143,4 @@ - \ No newline at end of file + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f59d773f33..27ef9330a7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,93 +1,93 @@ - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c - + https://github.com/dotnet/dnceng-shared - 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 + c4ed8b69bcb0a344a9fe2004a5944ca3a24bec1c diff --git a/eng/Versions.props b/eng/Versions.props index f8d27f6951..954dbe9ea0 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,28 +15,28 @@ 8.0.0-beta.24352.1 8.0.0-beta.24352.1 17.4.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 - 1.1.0-beta.24353.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 + 1.1.0-beta.24359.1 1.1.0-beta.24351.3 1.1.0-beta.24351.3 From 78939636941717922c23c0c73279602159853de6 Mon Sep 17 00:00:00 2001 From: Oleksandr Didyk <106967057+oleksandr-didyk@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:59:56 +0200 Subject: [PATCH 04/14] Fixup AzDO client construction in DI (#3717) --- .../Darc/Operations/Operation.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs index 78008d0a9f..29aa0057fb 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs @@ -66,9 +66,16 @@ protected Operation(ICommandLineOptions options, IServiceCollection? services = DisableInteractiveAuth = options.IsCi, }; }); - services.TryAddSingleton(); - services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(s => + new AzureDevOpsClient( + s.GetRequiredService(), + s.GetRequiredService(), + s.GetRequiredService()) + ); + services.TryAddSingleton(s => + s.GetRequiredService() + ); Provider = services.BuildServiceProvider(); Logger = Provider.GetRequiredService>(); From 3efd2839e7ebf5084a84a6e1b8d50acd489ffc45 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:34:14 +0200 Subject: [PATCH 05/14] [main] Update dependencies from dotnet/arcade (#3719) Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 24 +++++++++---------- eng/Versions.props | 10 ++++---- .../job/publish-build-assets.yml | 9 ++++--- .../templates/job/publish-build-assets.yml | 9 ++++--- global.json | 2 +- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 27ef9330a7..79d4e01ed6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -91,29 +91,29 @@ - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 954dbe9ea0..13dce6766b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 - 8.0.0-beta.24352.1 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 + 8.0.0-beta.24360.5 17.4.1 1.1.0-beta.24359.1 1.1.0-beta.24359.1 diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index d01739c128..ba3e7df815 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -140,11 +140,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9fd69fa7c9..57a41f0a3e 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -136,11 +136,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/global.json b/global.json index 4a8f4fcce4..4256a85777 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" } } From 8443ad9779539e3619d004b1163e9f58baefe3a1 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:43:53 +0200 Subject: [PATCH 06/14] Revert hardcoded darc (#3726) --- eng/common/post-build/publish-using-darc.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index e1a1760022..238945cb5a 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -11,7 +11,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc "1.1.0-beta.24361.11" + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() From eda3437d7ec61723ed40ff99ab838329a3a62d39 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:17:26 +0200 Subject: [PATCH 07/14] Add darc 'add-build-to-channel' test (#3725) --- eng/get-darc-version.ps1 | 12 ------------ eng/templates/stages/deploy.yaml | 28 ++++++++++++++++++---------- 2 files changed, 18 insertions(+), 22 deletions(-) delete mode 100644 eng/get-darc-version.ps1 diff --git a/eng/get-darc-version.ps1 b/eng/get-darc-version.ps1 deleted file mode 100644 index bfde0d76af..0000000000 --- a/eng/get-darc-version.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -param( - [Parameter(Mandatory=$True)] - [string]$maestroTestEndpoints, - [Parameter(Mandatory=$True)] - [string]$apiVersion -) - -$maestroTestEndpoint = $maestroTestEndpoints.Split(',')[0] -$versionEndpoint = "$maestroTestEndpoint/api/assets/darc-version?api-version=$apiVersion" -$latestDarcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content -Write-Host "##vso[task.setvariable variable=darcVersion]$latestDarcVersion" -Write-Host "Using Darc version $latestDarcVersion to run the tests" \ No newline at end of file diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index e34d8ee128..1ea30f6052 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -203,6 +203,24 @@ stages: .\darc\darc.exe get-default-channels --source-repo arcade-services --ci -t "$(GetAuthInfo.FederatedToken)" --bar-uri "$(GetAuthInfo.BarUri)" --debug displayName: Test Federated token authentication + - ${{ if in(variables['Build.SourceBranch'], 'refs/heads/main', 'refs/heads/production') }}: + - powershell: + $darcBuild = .\darc\darc.exe get-build ` + --repo "https://github.com/dotnet/arcade-services" ` + --commit $(Build.SourceVersion) ` + --ci ` + -p $(GetAuthInfo.Token) ` + --output-format json | + ConvertFrom-Json + + .\darc\darc.exe add-build-to-channel ` + --id $darcBuild[0].id ` + --channel "General Testing" + --ci ` + -p $(GetAuthInfo.Token) ` + --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) ` + displayName: Test Darc add-build-to-channel + - task: VSTest@2 displayName: Maestro Scenario Tests inputs: @@ -219,13 +237,3 @@ stages: DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts DARC_DIR: $(Build.SourcesDirectory)\darc DARC_IS_CI: true - - - powershell: | - nuget sources add -Name "arcade" -Source "https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json" - nuget sources add -Name "dotnet-core" -Source "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" - displayName: Add nuget Sources - - - powershell: $(Build.SourcesDirectory)\eng\get-darc-version.ps1 - -maestroTestEndpoints "${{ parameters.MaestroTestEndpoints }}" - -apiVersion "2019-01-16" - displayName: Get DARC version From f7926c9b78a29b71b55d9c72d068f20d41af45df Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:54:55 +0200 Subject: [PATCH 08/14] Pin System.Formats.Asn1 to 8.0.1 (#3727) --- Directory.Packages.props | 1 + 1 file changed, 1 insertion(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index 9b6391b015..8f7bd4cce9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -134,6 +134,7 @@ + From 82e953841c6a60dc82d2a9a3fb8a63442b767d1f Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Tue, 16 Jul 2024 14:58:38 +0200 Subject: [PATCH 09/14] Use prod darc to validate darc (#3728) --- eng/templates/stages/deploy.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 1ea30f6052..165cbeb6f9 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -204,12 +204,17 @@ stages: displayName: Test Federated token authentication - ${{ if in(variables['Build.SourceBranch'], 'refs/heads/main', 'refs/heads/production') }}: - - powershell: + - task: AzureCLI@2 + displayName: Test Darc add-build-to-channel + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | $darcBuild = .\darc\darc.exe get-build ` --repo "https://github.com/dotnet/arcade-services" ` --commit $(Build.SourceVersion) ` --ci ` - -p $(GetAuthInfo.Token) ` --output-format json | ConvertFrom-Json @@ -217,8 +222,7 @@ stages: --id $darcBuild[0].id ` --channel "General Testing" --ci ` - -p $(GetAuthInfo.Token) ` - --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) ` + --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) displayName: Test Darc add-build-to-channel - task: VSTest@2 From d64f018f793e62a642bae7c3f37ec69e578c7414 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Tue, 16 Jul 2024 15:38:10 +0200 Subject: [PATCH 10/14] Fix pipeline indentation (#3729) --- eng/templates/stages/deploy.yaml | 35 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 165cbeb6f9..3507e1f68e 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -205,25 +205,24 @@ stages: - ${{ if in(variables['Build.SourceBranch'], 'refs/heads/main', 'refs/heads/production') }}: - task: AzureCLI@2 - displayName: Test Darc add-build-to-channel - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - $darcBuild = .\darc\darc.exe get-build ` - --repo "https://github.com/dotnet/arcade-services" ` - --commit $(Build.SourceVersion) ` - --ci ` - --output-format json | - ConvertFrom-Json - - .\darc\darc.exe add-build-to-channel ` - --id $darcBuild[0].id ` - --channel "General Testing" - --ci ` - --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) displayName: Test Darc add-build-to-channel + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + $darcBuild = .\darc\darc.exe get-build ` + --repo "https://github.com/dotnet/arcade-services" ` + --commit $(Build.SourceVersion) ` + --ci ` + --output-format json | + ConvertFrom-Json + + .\darc\darc.exe add-build-to-channel ` + --id $darcBuild[0].id ` + --channel "General Testing" + --ci ` + --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) - task: VSTest@2 displayName: Maestro Scenario Tests From 38fd914311cd2ec406a99c3e00eaf833de20f31f Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 17 Jul 2024 10:51:59 +0200 Subject: [PATCH 11/14] Fix yaml (#3730) --- eng/templates/stages/deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index 3507e1f68e..a15ec04f12 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -213,14 +213,14 @@ stages: inlineScript: | $darcBuild = .\darc\darc.exe get-build ` --repo "https://github.com/dotnet/arcade-services" ` - --commit $(Build.SourceVersion) ` + --commit "$(Build.SourceVersion)" ` --ci ` --output-format json | ConvertFrom-Json .\darc\darc.exe add-build-to-channel ` --id $darcBuild[0].id ` - --channel "General Testing" + --channel "General Testing" ` --ci ` --azdev-pat $(dn-bot-dnceng-build-rw-code-rw-release-rw) From b28e59af12cd5e5e07d7c84c2ae251f88b3107b4 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:10:00 +0200 Subject: [PATCH 12/14] Register IRemoteTokenProvider (#3731) --- src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs index 29aa0057fb..e4fe31f7f6 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Maestro.Common; using Maestro.Common.AzureDevOpsTokens; using Microsoft.Arcade.Common; using Microsoft.DotNet.Darc.Helpers; @@ -76,7 +77,7 @@ protected Operation(ICommandLineOptions options, IServiceCollection? services = services.TryAddSingleton(s => s.GetRequiredService() ); - + services.TryAddSingleton(new RemoteTokenProvider(options.AzureDevOpsPat, options.GitHubPat)); Provider = services.BuildServiceProvider(); Logger = Provider.GetRequiredService>(); options.InitializeFromSettings(Logger); From 9d6a955597b3e5081a2ad82dc0d80eaf9f3359c7 Mon Sep 17 00:00:00 2001 From: Djuradj Kurepa <91743470+dkurepa@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:51:56 +0200 Subject: [PATCH 13/14] Initialize IRemoteProvider when it's needed, not during DI registration (#3732) --- src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs index e4fe31f7f6..ec4b81b84a 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs @@ -77,7 +77,8 @@ protected Operation(ICommandLineOptions options, IServiceCollection? services = services.TryAddSingleton(s => s.GetRequiredService() ); - services.TryAddSingleton(new RemoteTokenProvider(options.AzureDevOpsPat, options.GitHubPat)); + services.TryAddSingleton(_ => new RemoteTokenProvider(options.AzureDevOpsPat, options.GitHubPat)); + Provider = services.BuildServiceProvider(); Logger = Provider.GetRequiredService>(); options.InitializeFromSettings(Logger); From 47e3672c762970073e4282bd563233da86bcca3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Wed, 17 Jul 2024 16:16:43 +0200 Subject: [PATCH 14/14] Add option to generate common CredScan suppression file for VMR sync (#3698) --- .../VirtualMonoRepo/InitializeOperation.cs | 1 + .../VirtualMonoRepo/UpdateOperation.cs | 1 + .../VmrSyncCommandLineOptions.cs | 3 + .../CredScanSuppressionsGenerator.cs | 209 ++++++++++++++++++ .../VirtualMonoRepo/IVmrInitializer.cs | 2 + .../DarcLib/VirtualMonoRepo/IVmrUpdater.cs | 2 + .../VirtualMonoRepo/VmrForwardFlower.cs | 3 + .../DarcLib/VirtualMonoRepo/VmrInfo.cs | 2 + .../DarcLib/VirtualMonoRepo/VmrInitializer.cs | 7 +- .../DarcLib/VirtualMonoRepo/VmrManagerBase.cs | 9 + .../VirtualMonoRepo/VmrRegistrations.cs | 1 + .../DarcLib/VirtualMonoRepo/VmrUpdater.cs | 10 +- .../VmrSyncRepoChangesTest.cs | 69 +++++- .../VmrTestsBase.cs | 14 +- 14 files changed, 321 insertions(+), 12 deletions(-) create mode 100644 src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs index c3ae7228ba..135fc8f0dd 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/InitializeOperation.cs @@ -38,6 +38,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs index 35d140b9d1..d1004fb698 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VirtualMonoRepo/UpdateOperation.cs @@ -36,6 +36,7 @@ protected override async Task ExecuteInternalAsync( _options.ComponentTemplate, _options.TpnTemplate, _options.GenerateCodeowners, + _options.GenerateCredScanSuppressions, _options.DiscardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs index e81609cfd9..0c2216109b 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Options/VirtualMonoRepo/VmrSyncCommandLineOptions.cs @@ -28,6 +28,9 @@ internal abstract class VmrSyncCommandLineOptions : VmrCommandLineOptions, IBase [Option("generate-codeowners", Required = false, HelpText = "Generate a common CODEOWNERS file for all repositories.")] public bool GenerateCodeowners { get; set; } = false; + [Option("generate-credscansuppressions", Required = false, HelpText = "Generate a common .config/CredScanSuppressions.json file for all repositories.")] + public bool GenerateCredScanSuppressions { get; set; } = false; + [Option("discard-patches", Required = false, HelpText = "Delete .patch files created during the sync.")] public bool DiscardPatches { get; set; } = false; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs new file mode 100644 index 0000000000..8f8c86c768 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CredScanSuppressionsGenerator.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public interface ICredScanSuppressionsGenerator +{ + Task UpdateCredScanSuppressions(CancellationToken cancellationToken); +} + +public class CredScanSuppressionsGenerator : ICredScanSuppressionsGenerator +{ + private static readonly IReadOnlyCollection s_credScanSuppressionsLocations = new[] + { + new UnixPath(".config/" + VmrInfo.CredScanSuppressionsFileName), + new UnixPath("eng/" + VmrInfo.CredScanSuppressionsFileName), + }; + + private readonly IVmrInfo _vmrInfo; + private readonly ISourceManifest _sourceManifest; + private readonly ILocalGitClient _localGitClient; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _jsonOptions = new() + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip, + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + public CredScanSuppressionsGenerator( + IVmrInfo vmrInfo, + ISourceManifest sourceManifest, + ILocalGitClient localGitClient, + IFileSystem fileSystem, + ILogger logger) + { + _vmrInfo = vmrInfo; + _sourceManifest = sourceManifest; + _localGitClient = localGitClient; + _fileSystem = fileSystem; + _logger = logger; + } + + /// + /// Generates the CredScanSuppressions.json file by gathering individual repo CredScanSuppressions.json files. + /// + public async Task UpdateCredScanSuppressions(CancellationToken cancellationToken) + { + _logger.LogInformation("Updating {credscansuppressions}...", VmrInfo.CredScanSuppressionsPath); + + var destPath = _vmrInfo.VmrPath / VmrInfo.CredScanSuppressionsPath; + + CredScanSuppressionFile vmrCredScanSuppressionsFile = new CredScanSuppressionFile(); + + _fileSystem.CreateDirectory(_fileSystem.GetDirectoryName(destPath) + ?? throw new Exception($"Failed to create {VmrInfo.CredScanSuppressionsFileName} in {destPath}")); + + bool fileExistedBefore = _fileSystem.FileExists(destPath); + + using (var destStream = _fileSystem.GetFileStream(destPath, FileMode.Create, FileAccess.Write)) + { + foreach (ISourceComponent component in _sourceManifest.Repositories.OrderBy(m => m.Path)) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, component.Path, cancellationToken); + + foreach (var submodule in _sourceManifest.Submodules.Where(s => s.Path.StartsWith($"{component.Path}/"))) + { + await AddCredScanSuppressionsContent(vmrCredScanSuppressionsFile, submodule.Path, cancellationToken); + } + } + + JsonSerializer.Serialize(destStream, vmrCredScanSuppressionsFile, _jsonOptions); + } + + if (vmrCredScanSuppressionsFile.Suppressions.Count == 0) + { + _fileSystem.DeleteFile(destPath); + } + + bool fileExistsAfter = _fileSystem.FileExists(destPath); + + if (fileExistsAfter || fileExistedBefore) + { + await _localGitClient.StageAsync(_vmrInfo.VmrPath, new string[] { VmrInfo.CredScanSuppressionsPath }, cancellationToken); + } + + _logger.LogInformation("{credscansuppressions} updated", VmrInfo.CredScanSuppressionsPath); + } + + private async Task AddCredScanSuppressionsContent(CredScanSuppressionFile vmrCredScanSuppressionsFile, string repoPath, CancellationToken cancellationToken) + { + // CredScanSuppressions.json files are very restricted in size so we can safely work with them in-memory + var content = new List(); + + foreach (var location in s_credScanSuppressionsLocations) + { + cancellationToken.ThrowIfCancellationRequested(); + + var repoCredScanSuppressionsPath = _vmrInfo.VmrPath / VmrInfo.SourcesDir / repoPath / location; + + if (!_fileSystem.FileExists(repoCredScanSuppressionsPath)) continue; + + var repoCredScanSuppressionsFile = JsonSerializer.Deserialize(await _fileSystem.ReadAllTextAsync(repoCredScanSuppressionsPath), _jsonOptions); + + if (repoCredScanSuppressionsFile != null && repoCredScanSuppressionsFile.Suppressions != null) + { + foreach (var suppression in repoCredScanSuppressionsFile.Suppressions) + { + if (suppression.File != null) + { + for (int i = 0; i < suppression.File.Count; i++) + { + suppression.File[i] = FixCredScanSuppressionsRule(repoPath, suppression.File[i]); + } + } + } + + vmrCredScanSuppressionsFile.Suppressions.AddRange(repoCredScanSuppressionsFile.Suppressions); + } + } + } + + /// + /// Fixes a CredScanSuppressions.json file rule by prefixing the path with the VMR location and replacing backslash. + /// + private static string FixCredScanSuppressionsRule(string repoPath, string file) + { + if (string.IsNullOrWhiteSpace(file)) + { + return file; + } + + if (file.Contains('\\')) + { + file = file.Replace('\\', '/'); + } + + return $"/{VmrInfo.SourcesDir}/{repoPath}{(file.StartsWith('/') ? string.Empty : '/')}{file}"; + } +} + +class CredScanSuppressionFile +{ + [JsonPropertyName("tool")] + public string Tool { get; set; } = "Credential Scanner"; + [JsonPropertyName("suppressions")] + public List Suppressions { get; set; } = []; +} + +class CredScanSuppression +{ + [JsonPropertyName("_justification")] + public string Justification { get; set; } = ""; + [JsonPropertyName("placeholder")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? Placeholder { get; set; } + [JsonPropertyName("file")] + [JsonConverter(typeof(SingleStringOrArrayConverter))] + public List? File { get; set; } +} + +class SingleStringOrArrayConverter : JsonConverter> +{ + public override List? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.Null: + return null; + case JsonTokenType.StartArray: + var list = new List(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndArray) + break; + var arrayItem = JsonSerializer.Deserialize(ref reader, options); + if (arrayItem != null) + { + list.Add(arrayItem); + } + } + return list; + default: + var item = JsonSerializer.Deserialize(ref reader, options); + return item != null ? [item] : null; + } + } + + public override void Write(Utf8JsonWriter writer, List objectToWrite, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, objectToWrite, objectToWrite.GetType(), options); + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs index 86acd1cdfb..d79a3710fc 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrInitializer.cs @@ -23,6 +23,7 @@ public interface IVmrInitializer /// Path to VMR's README.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used Task InitializeRepository( string mappingName, @@ -34,6 +35,7 @@ Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs index 7f73283bf9..27bcd99faf 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/IVmrUpdater.cs @@ -22,6 +22,7 @@ public interface IVmrUpdater /// Path to VMR's Component.md template /// Path to VMR's THIRD-PARTY-NOTICES.md template /// Whether to generate a CODEOWNERS file + /// Whether to generate a .config/CredScanSuppressions.json file /// Whether to clean up genreated .patch files after their used /// True if the repository was updated, false if it was already up to date Task UpdateRepository( @@ -33,6 +34,7 @@ Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs index 2d1648791e..bbeafc5123 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs @@ -210,6 +210,7 @@ protected override async Task SameDirectionFlowAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: true, + generateCredScanSuppressions: true, discardPatches, cancellationToken); } @@ -255,6 +256,7 @@ await FlowCodeAsync( componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } @@ -330,6 +332,7 @@ .. submodules.Select(s => s.Path).Distinct().Select(VmrPatchHandler.GetExclusion componentTemplatePath: null, tpnTemplatePath: null, generateCodeowners: false, + generateCredScanSuppressions: false, discardPatches, cancellationToken); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs index 071c343544..1ac0391797 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs @@ -65,6 +65,7 @@ public class VmrInfo : IVmrInfo { public static readonly UnixPath SourcesDir = new("src"); public static readonly UnixPath CodeownersPath = new(".github/" + CodeownersFileName); + public static readonly UnixPath CredScanSuppressionsPath = new(".config/" + CredScanSuppressionsFileName); public const string SourceMappingsFileName = "source-mappings.json"; public const string GitInfoSourcesDir = "prereqs/git-info"; @@ -77,6 +78,7 @@ public class VmrInfo : IVmrInfo public const string ComponentListPath = "Components.md"; public const string ThirdPartyNoticesFileName = "THIRD-PARTY-NOTICES.txt"; public const string CodeownersFileName = "CODEOWNERS"; + public const string CredScanSuppressionsFileName = "CredScanSuppressions.json"; public static UnixPath RelativeSourcesDir { get; } = new("src"); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs index ef177c9cb3..a3394e292b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInitializer.cs @@ -55,6 +55,7 @@ public VmrInitializer( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -63,7 +64,7 @@ public VmrInitializer( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _vmrInfo = vmrInfo; _dependencyTracker = dependencyTracker; @@ -84,6 +85,7 @@ public async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -139,6 +141,7 @@ await InitializeRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -168,6 +171,7 @@ private async Task InitializeRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -206,6 +210,7 @@ await UpdateRepoToRevisionAsync( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs index cbdfc0104c..63d06f72d1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrManagerBase.cs @@ -30,6 +30,7 @@ public abstract class VmrManagerBase private readonly IThirdPartyNoticesGenerator _thirdPartyNoticesGenerator; private readonly IComponentListGenerator _componentListGenerator; private readonly ICodeownersGenerator _codeownersGenerator; + private readonly ICredScanSuppressionsGenerator _credScanSuppressionsGenerator; private readonly ILocalGitClient _localGitClient; private readonly IDependencyFileManager _dependencyFileManager; private readonly IFileSystem _fileSystem; @@ -46,6 +47,7 @@ protected VmrManagerBase( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator componentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -61,6 +63,7 @@ protected VmrManagerBase( _thirdPartyNoticesGenerator = thirdPartyNoticesGenerator; _componentListGenerator = componentListGenerator; _codeownersGenerator = codeownersGenerator; + _credScanSuppressionsGenerator = credScanSuppressionsGenerator; _localGitClient = localGitClient; _dependencyFileManager = dependencyFileManager; _fileSystem = fileSystem; @@ -79,6 +82,7 @@ public async Task> UpdateRepoToRevisionAs string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -141,6 +145,11 @@ public async Task> UpdateRepoToRevisionAs await _codeownersGenerator.UpdateCodeowners(cancellationToken); } + if (generateCredScanSuppressions) + { + await _credScanSuppressionsGenerator.UpdateCredScanSuppressions(cancellationToken); + } + // Commit without adding files as they were added to index directly await CommitAsync(commitMessage, author); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index fa4b0f1d86..cce0398af8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -63,6 +63,7 @@ public static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs index d33d305eb7..792969e0ae 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrUpdater.cs @@ -67,6 +67,7 @@ public VmrUpdater( IThirdPartyNoticesGenerator thirdPartyNoticesGenerator, IComponentListGenerator readmeComponentListGenerator, ICodeownersGenerator codeownersGenerator, + ICredScanSuppressionsGenerator credScanSuppressionsGenerator, ILocalGitClient localGitClient, ILocalGitRepoFactory localGitRepoFactory, IDependencyFileManager dependencyFileManager, @@ -76,7 +77,7 @@ public VmrUpdater( ILogger logger, ISourceManifest sourceManifest, IVmrInfo vmrInfo) - : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, patchHandler, versionDetailsParser, thirdPartyNoticesGenerator, readmeComponentListGenerator, codeownersGenerator, credScanSuppressionsGenerator, localGitClient, localGitRepoFactory, dependencyFileManager, fileSystem, logger) { _logger = logger; _sourceManifest = sourceManifest; @@ -101,6 +102,7 @@ public async Task UpdateRepository( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -132,6 +134,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -146,6 +149,7 @@ public async Task UpdateRepository( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); return true; @@ -165,6 +169,7 @@ private async Task> UpdateRepositoryInter string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -244,6 +249,7 @@ await UpdateTargetVersionOnly( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } @@ -258,6 +264,7 @@ private async Task UpdateRepositoryRecursively( string? componentTemplatePath, string? tpnTemplatePath, bool generateCodeowners, + bool generateCredScanSuppressions, bool discardPatches, CancellationToken cancellationToken) { @@ -336,6 +343,7 @@ private async Task UpdateRepositoryRecursively( componentTemplatePath, tpnTemplatePath, generateCodeowners, + generateCredScanSuppressions, discardPatches, cancellationToken); } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs index e20a2d6f2b..b941979689 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrSyncRepoChangesTest.cs @@ -146,8 +146,10 @@ public async Task SubmodulesAreInlinedProperlyTest() await GitOperations.InitializeSubmodule(ProductRepoPath, submoduleName, SecondRepoPath, submoduleRelativePath); Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CodeownersPath, "# This is a first repo's CODEOWNERS\nfoo/bar @some/team"); + Directory.CreateDirectory(Path.GetDirectoryName(ProductRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(ProductRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test"", ""file"": ""testfile"" } ] }"); await GitOperations.CommitAll(ProductRepoPath, "Add submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); var expectedFilesFromRepos = new List { @@ -155,6 +157,7 @@ public async Task SubmodulesAreInlinedProperlyTest() _dependencyRepoFilePath, submoduleFilePath, VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CodeownersPath, + VmrPath / VmrInfo.SourcesDir / Constants.ProductRepoName / VmrInfo.CredScanSuppressionsPath, }; List expectedFiles = GetExpectedFilesInVmr( @@ -163,6 +166,7 @@ public async Task SubmodulesAreInlinedProperlyTest() expectedFilesFromRepos); expectedFiles.Add(VmrPath / VmrInfo.CodeownersPath); + expectedFiles.Add(VmrPath / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CompareFileContents(_productRepoFilePath, _productRepoFileName); @@ -179,6 +183,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -187,14 +207,17 @@ public async Task SubmodulesAreInlinedProperlyTest() await File.WriteAllTextAsync(SecondRepoPath / additionalFileName, "New external repo file"); Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CodeownersPath)!); await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CodeownersPath, "# This is a second repo's CODEOWNERS\n/xyz/foo @other/team"); + Directory.CreateDirectory(Path.GetDirectoryName(SecondRepoPath / VmrInfo.CredScanSuppressionsPath)!); + await File.WriteAllTextAsync(SecondRepoPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ { ""_justification"": ""test2"", ""file"": ""testfile2"" } ] }"); await GitOperations.CommitAll(SecondRepoPath, "Adding new file in the submodule"); await GitOperations.PullMain(ProductRepoPath / submoduleRelativePath); await GitOperations.CommitAll(ProductRepoPath, "Checkout submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Add(additionalSubmoduleFilePath); expectedFiles.Add(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Add(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); CheckFileContents( @@ -214,18 +237,42 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/externals/product-repo2/xyz/foo @other/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + }, + { + "_justification": "test2", + "file": [ + "/src/product-repo1/externals/product-repo2/testfile2" + ] + } + ] + } + """, + removeEmptyLines: false); await GitOperations.CheckAllIsCommitted(VmrPath); // Remove submodule await GitOperations.RemoveSubmodule(ProductRepoPath, submoduleRelativePath); await File.WriteAllTextAsync(VmrPath / VmrInfo.CodeownersPath, "My new content in the CODEOWNERS\n\n### CONTENT BELOW IS AUTO-GENERATED AND MANUAL CHANGES WILL BE OVERWRITTEN ###\n"); + await File.WriteAllTextAsync(VmrPath / VmrInfo.CredScanSuppressionsPath, @"{ ""tool"": ""Credential Scanner"", ""suppressions"": [ ] }"); await GitOperations.CommitAll(ProductRepoPath, "Remove the submodule"); - await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true); + await UpdateRepoToLastCommit(Constants.ProductRepoName, ProductRepoPath, generateCodeowners: true, generateCredScanSuppressions: true); expectedFiles.Remove(submoduleFilePath); expectedFiles.Remove(additionalSubmoduleFilePath); expectedFiles.Remove(submodulePathInVmr / VmrInfo.CodeownersPath); + expectedFiles.Remove(submodulePathInVmr / VmrInfo.CredScanSuppressionsPath); CheckDirectoryContents(VmrPath, expectedFiles); await GitOperations.CheckAllIsCommitted(VmrPath); @@ -243,6 +290,22 @@ public async Task SubmodulesAreInlinedProperlyTest() /src/product-repo1/foo/bar @some/team """, removeEmptyLines: false); + CheckFileContents( + VmrPath / VmrInfo.CredScanSuppressionsPath, + """ + { + "tool": "Credential Scanner", + "suppressions": [ + { + "_justification": "test", + "file": [ + "/src/product-repo1/testfile" + ] + } + ] + } + """, + removeEmptyLines: false); } protected override async Task CopyReposForCurrentTest() diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs index 8ecb3c196b..fe40bfdab9 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs @@ -160,29 +160,29 @@ protected async Task InitializeRepoAtLastCommit(string repoName, NativePath repo await CallDarcInitialize(repoName, commit, sourceMappings); } - protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false) + protected async Task UpdateRepoToLastCommit(string repoName, NativePath repoPath, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { var commit = await GitOperations.GetRepoLastCommit(repoPath); - await CallDarcUpdate(repoName, commit, generateCodeowners); + await CallDarcUpdate(repoName, commit, generateCodeowners, generateCredScanSuppressions); } private async Task CallDarcInitialize(string repository, string commit, LocalPath sourceMappingsPath) { using var scope = ServiceProvider.CreateScope(); var vmrInitializer = scope.ServiceProvider.GetRequiredService(); - await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, true, _cancellationToken.Token); + await vmrInitializer.InitializeRepository(repository, commit, null, true, sourceMappingsPath, Array.Empty(), null, null, false, false, true, _cancellationToken.Token); } - protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { - await CallDarcUpdate(repository, commit, [], generateCodeowners); + await CallDarcUpdate(repository, commit, [], generateCodeowners, generateCredScanSuppressions); } - protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false) + protected async Task CallDarcUpdate(string repository, string commit, AdditionalRemote[] additionalRemotes, bool generateCodeowners = false, bool generateCredScanSuppressions = false) { using var scope = ServiceProvider.CreateScope(); var vmrUpdater = scope.ServiceProvider.GetRequiredService(); - await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, true, _cancellationToken.Token); + await vmrUpdater.UpdateRepository(repository, commit, null, true, additionalRemotes, null, null, generateCodeowners, generateCredScanSuppressions, true, _cancellationToken.Token); } protected async Task CallDarcBackflow(string mappingName, NativePath repoPath, string branch, string? shaToFlow = null, int? buildToFlow = null)