From a399fa27d23406374f756e1124a0d915088389d9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 12:27:17 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20240531.1 (#2518) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 12 +- eng/Versions.props | 2 +- eng/common/SetupNugetSources.ps1 | 159 ++++++++---------- eng/common/SetupNugetSources.sh | 60 +++---- .../core-templates/job/source-build.yml | 13 ++ .../core-templates/jobs/source-build.yml | 8 + .../steps/enable-internal-runtimes.yml | 32 ++++ .../steps/enable-internal-sources.yml | 35 ++++ .../steps/get-delegation-sas.yml} | 3 + .../steps/get-federated-access-token.yml | 28 +++ .../core-templates/steps/source-build.yml | 12 +- eng/common/cross/armv6/sources.list.bookworm | 2 + eng/common/cross/build-rootfs.sh | 7 + .../steps/enable-internal-runtimes.yml | 9 + .../steps/enable-internal-sources.yml | 7 + .../steps/get-delegation-sas.yml | 7 + .../steps/get-federated-access-token.yml | 7 + .../steps/enable-internal-runtimes.yml | 10 ++ .../steps/enable-internal-sources.yml | 7 + .../templates/steps/get-delegation-sas.yml | 7 + .../steps/get-federated-access-token.yml | 7 + global.json | 4 +- 22 files changed, 299 insertions(+), 139 deletions(-) create mode 100644 eng/common/core-templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/core-templates/steps/enable-internal-sources.yml rename eng/common/{templates/steps/get-delegate-sas.yml => core-templates/steps/get-delegation-sas.yml} (96%) create mode 100644 eng/common/core-templates/steps/get-federated-access-token.yml create mode 100644 eng/common/cross/armv6/sources.list.bookworm create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/enable-internal-sources.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/enable-internal-sources.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c7bdce0a8..439d9af36 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,17 +3,17 @@ - + https://github.com/dotnet/arcade - 0c4e52c37a29b75f64646220e0cc237177ce23a2 + e6b3f32f9855dccbe2447471c8f729b66f17d242 - + https://github.com/dotnet/arcade - 0c4e52c37a29b75f64646220e0cc237177ce23a2 + e6b3f32f9855dccbe2447471c8f729b66f17d242 - + https://github.com/dotnet/arcade - 0c4e52c37a29b75f64646220e0cc237177ce23a2 + e6b3f32f9855dccbe2447471c8f729b66f17d242 diff --git a/eng/Versions.props b/eng/Versions.props index f50a102ba..3ab823d5c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,6 +15,6 @@ 8.0.0 8.0.0 8.0.0 - 9.0.0-beta.24278.1 + 9.0.0-beta.24281.1 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index efa2fd72b..c07f6a526 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -1,31 +1,32 @@ -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on internal package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. This credential +# is added via the standard environment variable VSS_NUGET_EXTERNAL_FEED_ENDPOINTS. See +# https://github.com/microsoft/artifacts-credprovider/tree/v1.1.1?tab=readme-ov-file#environment-variables for more details # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing +# See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials +# displayName: Setup Internal Feeds # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config +# - task: NuGetAuthenticate@1 +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed +# +# This logic is also abstracted into enable-internal-sources.yml. [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][string]$Password + [string]$Password ) $ErrorActionPreference = "Stop" @@ -34,12 +35,23 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 +$feedEndpoints = $null + +# If a credential is provided, ensure that we don't overwrite the current set of +# credentials that may have been provided by a previous call to the credential provider. +if ($Password -and $null -ne $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS) { + $feedEndpoints = $env:VSS_NUGET_EXTERNAL_FEED_ENDPOINTS | ConvertFrom-Json +} elseif ($Password) { + $feedEndpoints = @{ endpointCredentials = @() } +} + # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") - if ($packageSource -eq $null) + if ($null -eq $packageSource) { + Write-Host "`tAdding package source" $SourceName $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) @@ -48,58 +60,34 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd -} - -# Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { - # Looks for credential configuration for the given SourceName. Create it if none is found. - $sourceElement = $creds.SelectSingleNode($Source) - if ($sourceElement -eq $null) - { - $sourceElement = $doc.CreateElement($Source) - $creds.AppendChild($sourceElement) | Out-Null - } - # Add the node to the credential if none is found. - $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']") - if ($usernameElement -eq $null) - { - $usernameElement = $doc.CreateElement("add") - $usernameElement.SetAttribute("key", "Username") - $sourceElement.AppendChild($usernameElement) | Out-Null + if ($pwd) { + $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $SourceEndPoint -pwd $pwd } - $usernameElement.SetAttribute("value", $Username) +} - # Add the to the credential if none is found. - # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs. - # -> https://github.com/NuGet/Home/issues/5526 - $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']") - if ($passwordElement -eq $null) - { - $passwordElement = $doc.CreateElement("add") - $passwordElement.SetAttribute("key", "ClearTextPassword") - $sourceElement.AppendChild($passwordElement) | Out-Null +# Add a new feed endpoint credential +function AddCredential([array]$endpointCredentials, $source, $pwd) { + $endpointCredentials += @{ + endpoint = $source; + password = $pwd } - - $passwordElement.SetAttribute("value", $pwd) + return $endpointCredentials } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") +function InsertMaestroInternalFeedCredentials($Sources, $pwd) { + $maestroInternalSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd + ForEach ($PackageSource in $maestroInternalSources) { + Write-Host "`tAdding credential for Maestro's feed:" $PackageSource.Key + $feedEndpoints.endpointCredentials = AddCredential -endpointCredentials $feedEndpoints.endpointCredentials -source $PackageSource.value -pwd $pwd } } -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +function EnableInternalPackageSources($DisabledPackageSources) { + $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroInternalSources) { + Write-Host "`tEnsuring internal source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) } @@ -110,11 +98,6 @@ if (!(Test-Path $ConfigFile -PathType Leaf)) { ExitWithExitCode 1 } -if (!$Password) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT' - ExitWithExitCode 1 -} - # Load NuGet.config $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName @@ -122,35 +105,27 @@ $doc.Load($filename) # Get reference to or create one if none exist already $sources = $doc.DocumentElement.SelectSingleNode("packageSources") -if ($sources -eq $null) { +if ($null -eq $sources) { $sources = $doc.CreateElement("packageSources") $doc.DocumentElement.AppendChild($sources) | Out-Null } -# Looks for a node. Create it if none is found. -$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") -if ($creds -eq $null) { - $creds = $doc.CreateElement("packageSourceCredentials") - $doc.DocumentElement.AppendChild($creds) | Out-Null -} - # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") -if ($disabledSources -ne $null) { +if ($null -ne $disabledSources) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources + EnableInternalPackageSources -DisabledPackageSources $disabledSources } -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password +if ($Password) { + InsertMaestroInternalFeedCredentials -Sources $sources -pwd $Password +} # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password +if ($null -ne $dotnet31Source) { + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json" -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json" -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -158,10 +133,18 @@ $dotnetVersions = @('5','6','7','8') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") - if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + if ($dotnetSource) { + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedprefix-internal/nuget/v3/index.json" -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v3/index.json" -pwd $Password } } -$doc.Save($filename) \ No newline at end of file +$doc.Save($filename) + +# If any credentials were added or altered, update the VSS_NUGET_EXTERNAL_FEED_ENDPOINTS environment variable +if ($null -ne $feedEndpoints) { + # ci is set to true so vso logging commands will be used. + $ci = $true + Write-PipelineSetVariable -Name 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' -Value $($feedEndpoints | ConvertTo-Json) -IsMultiJobVariable $false + Write-PipelineSetVariable -Name 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED' -Value "False" -IsMultiJobVariable $false +} \ No newline at end of file diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index d387c7eac..16c1e29ea 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -1,28 +1,27 @@ #!/usr/bin/env bash -# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds. -# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080 +# This script adds internal feeds required to build commits that depend on intenral package sources. For instance, +# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables +# disabled internal Maestro (darc-int*) feeds. +# +# Optionally, this script also adds a credential entry for each of the internal feeds if supplied. # -# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry -# under for each Maestro's managed private feed. Two additional credential -# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport. -# -# This script needs to be called in every job that will restore packages and which the base repo has -# private AzDO feeds in the NuGet.config. -# -# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)` -# from the AzureDevOps-Artifact-Feeds-Pats variable group. -# -# Any disabledPackageSources entries which start with "darc-int" will be re-enabled as part of this script executing. +# See example call for this script below. # # - task: Bash@3 -# displayName: Setup Private Feeds Credentials +# displayName: Setup Internal Feeds # inputs: # filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(Build.SourcesDirectory)/NuGet.config $Token +# arguments: $(Build.SourcesDirectory)/NuGet.config # condition: ne(variables['Agent.OS'], 'Windows_NT') -# env: -# Token: $(dn-bot-dnceng-artifact-feeds-rw) +# - task: NuGetAuthenticate@1 +# +# Note that the NuGetAuthenticate task should be called after SetupNugetSources. +# This ensures that: +# - Appropriate creds are set for the added internal feeds (if not supplied to the scrupt) +# - The credential provider is installed. +# +# This logic is also abstracted into enable-internal-sources.yml. ConfigFile=$1 CredToken=$2 @@ -48,11 +47,6 @@ if [ ! -f "$ConfigFile" ]; then ExitWithExitCode 1 fi -if [ -z "$CredToken" ]; then - Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT" - ExitWithExitCode 1 -fi - if [[ `uname -s` == "Darwin" ]]; then NL=$'\\\n' TB='' @@ -140,18 +134,20 @@ PackageSources+="$IFS" PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"') IFS=$PrevIFS -for FeedName in ${PackageSources[@]} ; do - # Check if there is no existing credential for this FeedName - grep -i "<$FeedName>" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." +if [ "$CredToken" ]; then + for FeedName in ${PackageSources[@]} ; do + # Check if there is no existing credential for this FeedName + grep -i "<$FeedName>" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding credentials for $FeedName." - PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + PackageSourceCredentialsNodeFooter="" + NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" - sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile - fi -done + sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile + fi + done +fi # Re-enable any entries in disabledPackageSources where the feed name contains darc-int grep -i "" $ConfigFile diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index c0ce4b3c8..c4713c8b6 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -33,6 +33,12 @@ parameters: is1ESPipeline: '' + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -74,6 +80,13 @@ jobs: - ${{ if eq(parameters.is1ESPipeline, '') }}: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} - template: /eng/common/core-templates/steps/source-build.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml index d8e5d0085..a10ccfbee 100644 --- a/eng/common/core-templates/jobs/source-build.yml +++ b/eng/common/core-templates/jobs/source-build.yml @@ -23,6 +23,12 @@ parameters: is1ESPipeline: '' + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -41,6 +47,7 @@ jobs: is1ESPipeline: ${{ parameters.is1ESPipeline }} jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/core-templates/job/source-build.yml @@ -48,3 +55,4 @@ jobs: is1ESPipeline: ${{ parameters.is1ESPipeline }} jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/core-templates/steps/enable-internal-runtimes.yml b/eng/common/core-templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000..6bdbf62ac --- /dev/null +++ b/eng/common/core-templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,32 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true +- name: is1ESPipeline + type: boolean + default: false + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl + is1ESPipeline: ${{ parameters.is1ESPipeline }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000..80deddafb --- /dev/null +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -0,0 +1,35 @@ +parameters: +# This is the Azure federated service connection that we log into to get an access token. +- name: nugetFederatedServiceConnection + type: string + default: 'dnceng-artifacts-feeds-read' +- name: is1ESPipeline + type: boolean + default: false + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. + # If running on DevDiv, NuGetAuthenticate is not really an option. It's scoped to a single feed, and we have many feeds that + # may be added. Instead, we'll use the traditional approach (add cred to nuget.config), but use an account token. + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config + - task: NuGetAuthenticate@1 + - ${{ else }}: + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.nugetFederatedServiceConnection }} + outputVariableName: 'dnceng-artifacts-feeds-read-access-token' + - task: PowerShell@2 + displayName: Setup Internal Feeds + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + # This is required in certain scenarios to install the ADO credential provider. + # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others + # (e.g. dotnet msbuild). + - task: NuGetAuthenticate@1 diff --git a/eng/common/templates/steps/get-delegate-sas.yml b/eng/common/core-templates/steps/get-delegation-sas.yml similarity index 96% rename from eng/common/templates/steps/get-delegate-sas.yml rename to eng/common/core-templates/steps/get-delegation-sas.yml index c0e8f9131..d2901470a 100644 --- a/eng/common/templates/steps/get-delegate-sas.yml +++ b/eng/common/core-templates/steps/get-delegation-sas.yml @@ -16,6 +16,9 @@ parameters: - name: permissions type: string default: 'rl' +- name: is1ESPipeline + type: boolean + default: false steps: - task: AzureCLI@2 diff --git a/eng/common/core-templates/steps/get-federated-access-token.yml b/eng/common/core-templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000..c8c49cc0e --- /dev/null +++ b/eng/common/core-templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml index bea5c4187..16c778d92 100644 --- a/eng/common/core-templates/steps/source-build.yml +++ b/eng/common/core-templates/steps/source-build.yml @@ -19,18 +19,10 @@ steps: set -x df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. + # If file changes are detected, set CopyWipIntoInnerSourceBuildRepo to copy the WIP changes into the inner source build repo. internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + if ! git diff --quiet; then internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. # This only works if there is a username/email configured, which won't be the case in most CI runs. git config --get user.email diff --git a/eng/common/cross/armv6/sources.list.bookworm b/eng/common/cross/armv6/sources.list.bookworm new file mode 100644 index 000000000..101611352 --- /dev/null +++ b/eng/common/cross/armv6/sources.list.bookworm @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ bookworm main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ bookworm main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index a8e35df7c..7455dcb6a 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -314,6 +314,13 @@ while :; do bullseye) # Debian 11 __CodeName=bullseye + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi + ;; + bookworm) # Debian 12 + __CodeName=bookworm + if [[ -z "$__UbuntuRepo" ]]; then __UbuntuRepo="http://ftp.debian.org/debian/" fi diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000..f9dd238c6 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,9 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default +steps: +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/enable-internal-sources.yml b/eng/common/templates-official/steps/enable-internal-sources.yml new file mode 100644 index 000000000..e6d571822 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 000000000..c5a9c1f82 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 000000000..c8dcf6b81 --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 000000000..b21a8038c --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,10 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +steps: +- template: /eng/common/core-templates/steps/enable-internal-runtimes.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/enable-internal-sources.yml b/eng/common/templates/steps/enable-internal-sources.yml new file mode 100644 index 000000000..5f87e9abb --- /dev/null +++ b/eng/common/templates/steps/enable-internal-sources.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/enable-internal-sources.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 000000000..83760c979 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-delegation-sas.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 000000000..31e151d9d --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,7 @@ +steps: +- template: /eng/common/core-templates/steps/get-federated-access-token.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/global.json b/global.json index 575ff3c1b..fbd8f6f42 100644 --- a/global.json +++ b/global.json @@ -18,7 +18,7 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24278.1", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24278.1" + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24281.1", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24281.1" } }