From 9c985df92553b8cb022bca516042746ecd168df5 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Tue, 14 May 2024 19:33:42 +0200 Subject: [PATCH] Use storage upload task for non-token auth; add test mode (#1225) "az" doesn't contain a tool that seems to support non-SAS-token-based auth. It will authenticate broadly using the service connection, but actual storage copies are performed by grabbing the access key. So, use AzureFileCopy@6, which explicitly supports what we need to do. This task only runs on Windows, so switch the publish stages over from Linux. Add "publishExistingRunID" parameter for test runs. This has been done previously as a hard-coded temporary change, but publish is complex enough now to warrant a more usable and permanent debug feature. (cherry picked from commit 7262f0e6016f55e9fcaf7f8f0be9a80240eb9129) --- eng/pipeline/rolling-internal-pipeline.yml | 6 + eng/pipeline/stages/builders-to-stages.yml | 67 ++++----- .../stages/go-builder-matrix-stages.yml | 4 + eng/pipeline/stages/publish-stage.yml | 130 ++++++++++++------ 4 files changed, 132 insertions(+), 75 deletions(-) diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml index cb7ffc33151..8b6397e3b26 100644 --- a/eng/pipeline/rolling-internal-pipeline.yml +++ b/eng/pipeline/rolling-internal-pipeline.yml @@ -24,6 +24,11 @@ parameters: type: boolean default: false + - name: publishExistingRunID + displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.' + type: string + default: nil + variables: - template: variables/pool-providers.yml # MicroBuild configuration. @@ -62,4 +67,5 @@ extends: createSourceArchive: true createSymbols: true publish: true + publishExistingRunID: ${{ parameters.publishExistingRunID }} releaseVersion: ${{ parameters.releaseVersion }} diff --git a/eng/pipeline/stages/builders-to-stages.yml b/eng/pipeline/stages/builders-to-stages.yml index 56e53bcb9a0..90e7b8a489b 100644 --- a/eng/pipeline/stages/builders-to-stages.yml +++ b/eng/pipeline/stages/builders-to-stages.yml @@ -9,6 +9,8 @@ parameters: builders: [] # If true, publish build artifacts to blob storage. publish: false + # If changed to specify an existing pipeline run, skip build/sign and publish the existing run. + publishExistingRunID: 'nil' # If true, include a signing stage+job that depends on all 'buildandpack' builder jobs finishing. # 'official' is passed through into run-stage.yml, where it has other effects. official: false @@ -19,36 +21,37 @@ parameters: releaseVersion: 'nil' stages: - - ${{ each builder in parameters.builders }}: - - template: pool.yml - parameters: - inner: - template: run-stage.yml - parameters: - builder: ${{ builder }} - createSourceArchive: ${{ parameters.createSourceArchive }} - releaseVersion: ${{ parameters.releaseVersion }} - official: ${{ parameters.official }} - createSymbols: ${{ parameters.createSymbols }} - # Attempt to retry the build on Windows to mitigate flakiness: - # "Access Denied" during EXE copying and general flakiness during tests. - ${{ if eq(builder.os, 'windows') }}: - retryAttempts: [1, 2, 3, 4, "FINAL"] + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - ${{ each builder in parameters.builders }}: + - template: pool.yml + parameters: + inner: + template: run-stage.yml + parameters: + builder: ${{ builder }} + createSourceArchive: ${{ parameters.createSourceArchive }} + releaseVersion: ${{ parameters.releaseVersion }} + official: ${{ parameters.official }} + createSymbols: ${{ parameters.createSymbols }} + # Attempt to retry the build on Windows to mitigate flakiness: + # "Access Denied" during EXE copying and general flakiness during tests. + ${{ if eq(builder.os, 'windows') }}: + retryAttempts: [1, 2, 3, 4, "FINAL"] - - ${{ if eq(parameters.official, true) }}: - - template: pool.yml - parameters: - inner: - template: sign-stage.yml - parameters: - # This is not a builder, but provide partial builder info for agent selection. - builder: { os: windows, arch: amd64 } - official: ${{ parameters.official }} - # The list of builders to depend on and grab artifacts from. - builders: - - ${{ each builder in parameters.builders }}: - - ${{ if eq(builder.config, 'buildandpack') }}: - - ${{ builder }} + - ${{ if eq(parameters.official, true) }}: + - template: pool.yml + parameters: + inner: + template: sign-stage.yml + parameters: + # This is not a builder, but provide partial builder info for agent selection. + builder: { os: windows, arch: amd64 } + official: ${{ parameters.official }} + # The list of builders to depend on and grab artifacts from. + builders: + - ${{ each builder in parameters.builders }}: + - ${{ if eq(builder.config, 'buildandpack') }}: + - ${{ builder }} - ${{ if eq(parameters.publish, true) }}: - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}: @@ -58,16 +61,17 @@ stages: template: publish-stage.yml parameters: # This is not a builder, but provide partial builder info for agent selection. - builder: { os: linux, arch: amd64 } + builder: { os: windows, arch: amd64 } official: true public: true + publishExistingRunID: ${{ parameters.publishExistingRunID }} - template: pool.yml parameters: inner: template: publish-stage.yml parameters: - builder: { os: linux, arch: amd64 } + builder: { os: windows, arch: amd64 } official: true public: false builders: @@ -75,3 +79,4 @@ stages: - ${{ if eq(builder.config, 'buildandpack') }}: - ${{ builder }} publishSymbols: ${{ parameters.createSymbols }} + publishExistingRunID: ${{ parameters.publishExistingRunID }} diff --git a/eng/pipeline/stages/go-builder-matrix-stages.yml b/eng/pipeline/stages/go-builder-matrix-stages.yml index 516a935e572..d148fb002a7 100644 --- a/eng/pipeline/stages/go-builder-matrix-stages.yml +++ b/eng/pipeline/stages/go-builder-matrix-stages.yml @@ -40,6 +40,9 @@ parameters: - name: releaseVersion type: string default: 'nil' + - name: publishExistingRunID + type: string + default: 'nil' stages: - template: shorthand-builders-to-builders.yml @@ -48,6 +51,7 @@ stages: jobsParameters: official: ${{ parameters.official }} publish: ${{ parameters.publish }} + publishExistingRunID: ${{ parameters.publishExistingRunID }} createSourceArchive: ${{ parameters.createSourceArchive }} createSymbols: ${{ parameters.createSymbols }} releaseVersion: ${{ parameters.releaseVersion }} diff --git a/eng/pipeline/stages/publish-stage.yml b/eng/pipeline/stages/publish-stage.yml index d4ba3eb2991..1f34051884b 100644 --- a/eng/pipeline/stages/publish-stage.yml +++ b/eng/pipeline/stages/publish-stage.yml @@ -11,6 +11,10 @@ parameters: - name: pool type: object + - name: publishExistingRunID + type: string + default: 'nil' + # Unused. Declared so pool selection doesn't fail when trying to pass them. - name: builder type: object @@ -29,25 +33,25 @@ stages: displayName: Publish Public ${{ else }}: displayName: Publish Internal - dependsOn: Sign + ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + dependsOn: Sign + ${{ else }}: + dependsOn: [] jobs: - job: Publish pool: ${{ parameters.pool }} variables: - - ${{ if parameters.public }}: - - name: blobContainer + - name: blobContainer + ${{ if parameters.public }}: value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft' - - name: blobSASArg - value: --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' - - ${{ else }}: - - name: blobContainer + ${{ else }}: value: 'https://golangartifacts.blob.core.windows.net/microsoft' - - name: blobSASArg - value: '' # golangartifacts is set up with service connection auth. + - name: blobPrefix + value: '$(PublishBranchAlias)/$(Build.BuildNumber)' - name: blobDestinationUrl - value: '$(blobContainer)/$(PublishBranchAlias)/$(Build.BuildNumber)' + value: '$(blobContainer)/$(blobPrefix)' - group: go-storage @@ -75,8 +79,7 @@ stages: artifact: SymbolsInternal steps: - - template: ../steps/checkout-unix-task.yml - - template: ../steps/init-pwsh-task.yml + - template: ../steps/checkout-windows-task.yml - template: ../steps/init-submodule-task.yml - pwsh: | @@ -102,11 +105,25 @@ stages: Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch" displayName: Find publish branch alias - - download: current - artifact: Binaries Signed - # Filter out manifests added by 1ES pipeline template. - patterns: '!_manifest/**' - displayName: 'Download: Binaries Signed' + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - download: current + artifact: Binaries Signed + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + displayName: 'Download: Binaries Signed' + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download: Binaries Signed (Specific)' + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + runVersion: 'specific' + runId: ${{ parameters.publishExistingRunID }} + artifact: Binaries Signed + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + targetPath: '$(Pipeline.Workspace)/Binaries Signed' - pwsh: | eng/run.ps1 createbuildassetjson ` @@ -117,36 +134,61 @@ stages: -o '$(Pipeline.Workspace)/Binaries Signed/assets.json' displayName: 'Create build asset JSON' - - task: AzureCLI@2 - displayName: Upload to blob storage - inputs: - azureSubscription: GoLang - scriptType: bash - scriptLocation: inlineScript - # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one - # "from" argument, so we can't use the shell's wildcard expansion. - inlineScript: | - az storage copy -s '*' -d '$(blobDestinationUrl)' $(blobSASArg) - workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' - - - script: | - echo 'Generated links to artifacts in blob storage:' - echo '' - for f in *; do - echo "$(blobDestinationUrl)/$f" - done - displayName: Show uploaded URLs + - ${{ if parameters.public }}: + - task: AzureCLI@2 + displayName: Upload to blob storage + inputs: + azureSubscription: GoLang + scriptType: bash + scriptLocation: inlineScript + # Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one + # "from" argument, so we can't use the shell's wildcard expansion. + inlineScript: | + az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)' + workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' + - ${{ else }}: + - task: AzureFileCopy@6 + displayName: Upload to blob storage + inputs: + Destination: AzureBlob + azureSubscription: GoLang + storage: golangartifacts + ContainerName: microsoft + SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*' + BlobPrefix: $(blobPrefix) + + - pwsh: | + Write-Host 'Generated links to artifacts in blob storage:' + Write-Host '' + Get-ChildItem -File -Path '.' | %{ + Write-Host "$(blobDestinationUrl)/$($_.Name)" + } + displayName: Show expected uploaded URLs workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/' - ${{ if eq(parameters.publishSymbols, true) }}: - ${{ each builder in parameters.builders }}: - - download: current - artifact: Symbols ${{ builder.id }} - # Filter out manifests added by 1ES pipeline template. - patterns: '!_manifest/**' - displayName: 'Download: Symbols ${{ builder.id }}' - - - powershell: | + - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: + - download: current + artifact: Symbols ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + displayName: 'Download: Symbols ${{ builder.id }}' + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download: Symbols ${{ builder.id }} (Specific)' + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + runVersion: 'specific' + runId: ${{ parameters.publishExistingRunID }} + artifact: Symbols ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + targetPath: '$(Pipeline.Workspace)/Symbols ${{ builder.id }}' + + - pwsh: | $flatDir = "$(Pipeline.Workspace)/Symbols" New-Item $flatDir -ItemType Directory -ErrorAction Ignore @@ -158,7 +200,7 @@ stages: } Copy-Item $_.FullName $flatDir } - displayName: 'Copy to flat dir: ${{ builder.id }}' + displayName: 'Flatten: Symbols ${{ builder.id }}' workingDirectory: '$(Pipeline.Workspace)' - task: PublishSymbols@2 inputs: