diff --git a/eng/_util/cmd/build/build.go b/eng/_util/cmd/build/build.go index 03d39428e6..e87ab87609 100644 --- a/eng/_util/cmd/build/build.go +++ b/eng/_util/cmd/build/build.go @@ -138,6 +138,17 @@ func build(o *options) error { } fmt.Printf("---- Target platform: %v_%v\n", targetOS, targetArch) + // Setting GOROOT explicitly in the environment has not been necessary since Go 1.9 + // (https://go.dev/doc/go1.9#goroot), but a dev or build machine may still have it set. It + // interferes with attempts to run the built Go (such as when building the race runtime), so + // remove the explicit GOROOT if set. + if explicitRoot, ok := os.LookupEnv("GOROOT"); ok { + fmt.Printf("---- Removing explicit GOROOT from environment: %v\n", explicitRoot) + if err := os.Unsetenv("GOROOT"); err != nil { + return err + } + } + // The upstream build scripts in {repo-root}/src require your working directory to be src, or // they instantly fail. Change the current process dir so that we can run them. if err := os.Chdir("go/src"); err != nil { diff --git a/eng/_util/go-install.ps1 b/eng/_util/go-install.ps1 new file mode 100644 index 0000000000..29cc7fc9a1 --- /dev/null +++ b/eng/_util/go-install.ps1 @@ -0,0 +1,899 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Installs Microsoft Go +.DESCRIPTION + Installs the Microsoft Go toolset. + + Note that the intended use of this script is for Continuous Integration (CI) scenarios, where: + - The toolset needs to be installed without user interaction and without admin rights. + - The toolset installation doesn't need to persist across multiple CI runs. + Visit https://github.com/microsoft/go for a list of other ways to install Microsoft Go. + +.PARAMETER Version + Default: Latest + Download the specified version. Supports some aliases. Possible values: + - Latest - the most recent major version. + - Previous - the second most recent major version. + - 2-part version in format go1.A - represents a specific major version. + examples: go1.18, go1.23 + - 3-part version in format go1.A.B - latest revision of a specific release. + examples: go1.18.0, go1.23.1 + - 4-part version in format go1.A.B-C - a specific revision of Microsoft Go, immutable. + examples: go1.18.0-1, go1.23.1-3 + Microsoft Go doesn't publish prereleases, so they are not available. +.PARAMETER InstallDir + Path to where to install Microsoft Go. Note that if a directory is given, GOROOT is placed + directly in that directory. + Default: - a folder automatically selected inside LocalApplicationData as evaluated by PowerShell. + Example auto on Windows: C:\Users\myself\AppData\Local\microsoft-go\ + Example auto on Linux: /home/myself/.local/share/microsoft-go/ + If OS or Architecture are not , the path includes OS and Architecture. This avoids + overlapping installations but still allows for a shorter path for ordinary situations. +.PARAMETER OS + Default: - this value represents currently running OS + Operating system of prebuilt toolset binaries to be installed. + Possible values are: , windows, linux, darwin +.PARAMETER Architecture + Default: - this value represents currently running OS architecture + Architecture of prebuilt toolset binaries to be installed. + Possible values are: , amd64, x64, 386, x86, arm64, arm +.PARAMETER DryRun + If set, it will not perform installation. Instead, it displays what command line to use to + consistently install currently requested version of Microsoft Go. For example, if you specify + Version 'Latest', it will print a command with the specific 4-part version so this command can + be used deterministicly in a build script. + It also prints the location the binaries would have been installed to. +.PARAMETER NoPath + By default, this script will update the environment variable PATH for the current process to + include the binaries folder inside installation folder. + If set, it will print the binaries location but not set any environment variable. +.PARAMETER AzurePipelinePath + If set, it will print an Azure DevOps logging command that causes the Azure DevOps to update the + PATH environment variable of subsequent build steps to include the binaries folder. +.PARAMETER ProxyAddress + If set, it will use the proxy when making web requests +.PARAMETER ProxyUseDefaultCredentials + Default: false + Use default credentials when using ProxyAddress. +.PARAMETER ProxyBypassList + If set, when using ProxyAddress, this comma separated url list is passed to the underlying + HttpClientHandler. +.PARAMETER DownloadTimeout + Determines timeout duration in seconds for downloading the toolset file. + Default: 1200 seconds (20 minutes) +.PARAMETER KeepArchive + If set, the downloaded file is kept. +.PARAMETER ArchivePath + A path to use to store the toolset archive file, a zip or tar.gz. + Default: a generated random filename in the system's temporary directory. +.PARAMETER Help + Displays this help message. +.PARAMETER Verbose + Displays diagnostics information. +.EXAMPLE + go-install.ps1 + Installs the latest released Microsoft Go version. +.EXAMPLE + go-install.ps1 -Version Previous + Installs the latest version of the previous major (1.X) version of Microsoft Go. +#> +[cmdletbinding()] +param( + [string]$Version="Latest", + [Alias('i')][string]$InstallDir="", + [string]$OS="", + [string]$Architecture="", + [switch]$DryRun, + [switch]$NoPath, + [switch]$AzurePipelinePath, + [string]$ProxyAddress, + [switch]$ProxyUseDefaultCredentials, + [string[]]$ProxyBypassList=@(), + [int]$DownloadTimeout=1200, + [switch]$KeepArchive, + [string]$ArchivePath, + [switch]$Help +) + +Set-StrictMode -Version Latest +$ErrorActionPreference="Stop" +$ProgressPreference="SilentlyContinue" + +$MicrosoftGoInstallScriptVersion = "0.0.1" + +function Say($str) { + try { + Write-Host "go-install: $str" + } + catch { + # Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output + Write-Output "go-install: $str" + } +} + +function Say-Warning($str) { + try { + Write-Warning "go-install: $str" + } + catch { + # Some platforms cannot utilize Write-Warning (Azure Functions, for instance). Fall back to Write-Output + Write-Output "go-install: Warning: $str" + } +} + +# Writes a line with error style settings. +# Use this function to show a human-readable comment along with an exception. +function Say-Error($str) { + try { + # Write-Error is quite oververbose for the purpose of the function, let's write one line with error style settings. + $Host.UI.WriteErrorLine("go-install: $str") + } + catch { + Write-Output "go-install: Error: $str" + } +} + +function Say-Verbose($str) { + try { + Write-Verbose "go-install: $str" + } + catch { + # Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output + Write-Output "go-install: $str" + } +} + +function Measure-Action($name, $block) { + $time = Measure-Command $block + $totalSeconds = $time.TotalSeconds + Say-Verbose "⏱ Action '$name' took $totalSeconds seconds" +} + +function Get-Remote-File-Size($zipUri) { + try { + $response = Invoke-WebRequest -Uri $zipUri -Method Head + $fileSize = $response.Headers["Content-Length"] + if ((![string]::IsNullOrEmpty($fileSize))) { + Say "Remote file $zipUri size is $fileSize bytes." + + return $fileSize + } + } + catch { + Say-Verbose "Content-Length header was not extracted for $zipUri." + } + + return $null +} + +function Say-Invocation($Invocation) { + $command = $Invocation.MyCommand; + $args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ") + Say-Verbose "$command $args" +} + +function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [System.Threading.CancellationToken]$cancellationToken = [System.Threading.CancellationToken]::None, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) { + $Attempts = 0 + $local:startTime = $(get-date) + + while ($true) { + try { + return & $ScriptBlock + } + catch { + $Attempts++ + if (($Attempts -lt $MaxAttempts) -and -not $cancellationToken.IsCancellationRequested) { + Start-Sleep $SecondsBetweenAttempts + } + else { + $local:elapsedTime = $(get-date) - $local:startTime + if (($local:elapsedTime.TotalSeconds - $DownloadTimeout) -gt 0 -and -not $cancellationToken.IsCancellationRequested) { + throw New-Object System.TimeoutException("Failed to reach the server: connection timeout: default timeout is $DownloadTimeout second(s)"); + } + throw; + } + } + } +} + +function Get-Machine-Architecture() { + Say-Invocation $MyInvocation + + # Try the .NET API. If we don't get anything, this is probably PowerShell on Windows. + try { + $Architecture = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture + if ($Architecture) { + # Possible values: https://learn.microsoft.com/en-us/dotnet/api/system.runtime.interopservices.architecture + return $Architecture.ToString().ToLowerInvariant() + } + } + catch { + Say-Verbose "Failed to get the machine architecture using .NET API. Falling back to environment variables." + } + + # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems. + # To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432. + # PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE. + # Possible values: amd64, x64, x86, arm64, arm + if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null ) { + return $ENV:PROCESSOR_ARCHITEW6432 + } + + return $ENV:PROCESSOR_ARCHITECTURE +} + +function Get-CLIArchitecture-From-Architecture([string]$Architecture) { + Say-Invocation $MyInvocation + + if ($Architecture -eq "") { + $Architecture = Get-Machine-Architecture + } + + switch ($Architecture.ToLowerInvariant()) { + { ($_ -eq "amd64") -or ($_ -eq "x64") } { return "amd64" } + { ($_ -eq "386") -or ($_ -eq "x86") } { return "386" } + { $_ -eq "arm" } { return "armv6l" } + { $_ -eq "arm64" } { return "arm64" } + default { throw "Architecture '$Architecture' not supported. If you think this is a bug, report it at https://github.com/microsoft/go/issues" } + } +} + +function Get-CLIOS-From-OS([string]$OS) { + Say-Invocation $MyInvocation + + if (!(Test-Path -LiteralPath 'variable:IsWindows')) { + # If we don't have IsWindows, this is Windows PowerShell (powershell), not PowerShell Core (pwsh). + # So, we can't use the variable, but we know we're on Windows. + $IsWindows = $true + } + + if ($OS -eq "") { + if ($IsWindows -or [System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT) { + $OS = "windows" + } + elseif ($IsLinux) { + $OS = "linux" + } + elseif ($IsMacOS) { + $OS = "darwin" + } + else { + throw "Unable to automatically determine the OS." + } + } + + switch ($OS.ToLowerInvariant()) { + { $_ -eq "windows" } { return "windows" } + { $_ -eq "linux" } { return "linux" } + { $_ -eq "darwin" } { return "darwin" } + default { throw "OS '$OS' not supported. If you think this is a bug, report it at https://github.com/microsoft/go/issues" } + } +} + +function Get-GeneratedArchivePath([string]$CLIOS) { + Say-Invocation $MyInvocation + + $Extension = switch ($CLIOS) { + "windows" { ".zip" } + default { ".tar.gz" } + } + + return [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) + $Extension +} + +function Fetch-SupportedVersion([string]$StableKey) { + # Figure out what's latest by querying the list of release branches. + $ReleaseBranchData = DownloadJson "https://aka.ms/golang/release/latest/release-branch-links.json" + + # Find first thing in the array of objects where the key by name is true. + foreach ($branch in $ReleaseBranchData) { + if (Get-OrNull $branch $StableKey) { + return $branch.version + } + } + + throw "Failed to find a branch where '$StableKey' is true." +} + +function Get-NormalizedVersion([string]$Version) { + Say-Invocation $MyInvocation + + if ([string]::IsNullOrEmpty($Version)) { + return "" + } + switch ($Version.ToLowerInvariant()) { + { $_ -eq "latest" } { return Fetch-SupportedVersion -StableKey "latestStable" } + { $_ -eq "previous" } { return Fetch-SupportedVersion -StableKey "previousStable" } + { $_ -like "go1.*" } { return $_ } + default { throw "Version '$Version' not recognized. Missing 'go' prefix? If you think this is a bug, report it at https://github.com/microsoft/go/issues" } + } +} + +function Load-Assembly([string] $Assembly) { + try { + Add-Type -Assembly $Assembly | Out-Null + } + catch { + # On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd. + # Loading the base class assemblies is not unnecessary as the types will automatically get resolved. + } +} + +function GetHTTPResponse([Uri] $Uri, [bool]$HeaderOnly, [bool]$DisableRedirect) +{ + $cts = New-Object System.Threading.CancellationTokenSource + + $downloadScript = { + + $HttpClient = $null + + try { + # HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet. + Load-Assembly -Assembly System.Net.Http + + if (-not $ProxyAddress) { + try { + # Despite no proxy being explicitly specified, we may still be behind a default proxy + $DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy; + if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) { + if ($null -ne $DefaultProxy.GetProxy($Uri)) { + $ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString + } else { + $ProxyAddress = $null + } + $ProxyUseDefaultCredentials = $true + } + } + catch { + # Eat the exception and move forward as the above code is an attempt + # at resolving the DefaultProxy that may not have been a problem. + $ProxyAddress = $null + Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...") + } + } + + $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler + if ($ProxyAddress) { + $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{ + Address=$ProxyAddress; + UseDefaultCredentials=$ProxyUseDefaultCredentials; + BypassList = $ProxyBypassList; + } + } + if ($DisableRedirect) { + $HttpClientHandler.AllowAutoRedirect = $false + } + $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler + + # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out + # Defaulting to 20 minutes allows it to work over much slower connections. + $HttpClient.Timeout = New-TimeSpan -Seconds $DownloadTimeout + + if ($HeaderOnly){ + $completionOption = [System.Net.Http.HttpCompletionOption]::ResponseHeadersRead + } + else { + $completionOption = [System.Net.Http.HttpCompletionOption]::ResponseContentRead + } + + $Task = $HttpClient.GetAsync("$Uri", $completionOption).ConfigureAwait("false"); + $Response = $Task.GetAwaiter().GetResult(); + + if (($null -eq $Response) -or ((-not $HeaderOnly) -and (-not ($Response.IsSuccessStatusCode)))) { + # The feed credential is potentially sensitive info. Do not log FeedCredential to console output. + $DownloadException = [System.Exception] "Unable to download $Uri." + + if ($null -ne $Response) { + $DownloadException.Data["StatusCode"] = [int] $Response.StatusCode + $DownloadException.Data["ErrorMessage"] = "Unable to download $Uri. Returned HTTP status code: " + $DownloadException.Data["StatusCode"] + + if (404 -eq [int] $Response.StatusCode) { + $cts.Cancel() + } + } + + throw $DownloadException + } + + return $Response + } + catch [System.Net.Http.HttpRequestException] { + $DownloadException = [System.Exception] "Unable to download $Uri." + + # Pick up the exception message and inner exceptions' messages if they exist + $CurrentException = $PSItem.Exception + $ErrorMsg = $CurrentException.Message + "`r`n" + while ($CurrentException.InnerException) { + $CurrentException = $CurrentException.InnerException + $ErrorMsg += $CurrentException.Message + "`r`n" + } + + # Check if there is an issue concerning TLS. + if ($ErrorMsg -like "*SSL/TLS*") { + $ErrorMsg += "Ensure that TLS 1.2 or higher is enabled to use this script.`r`n" + } + + $DownloadException.Data["ErrorMessage"] = $ErrorMsg + throw $DownloadException + } + finally { + if ($null -ne $HttpClient) { + $HttpClient.Dispose() + } + } + } + + try { + return Invoke-With-Retry $downloadScript $cts.Token + } + finally { + if ($null -ne $cts) { + $cts.Dispose() + } + } +} + +function Resolve-Installation-Path([string]$InstallDir) { + Say-Invocation $MyInvocation + + if ($InstallDir -eq "") { + $Dir = Join-Path -Path ([Environment]::GetFolderPath('LocalApplicationData')) -ChildPath "microsoft-go" + if ($OS -ne "" -or $Architecture -ne "") { + $Dir = Join-Path -Path $Dir -ChildPath "$($CLIOS)_$CLIArchitecture" + } + return $Dir + } + return $InstallDir +} + +function Resolve-Versioned-Installation-Path([string]$InstallRoot, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + return Join-Path -Path $InstallRoot -ChildPath "go$SpecificVersion" +} + +function Is-ToolsetInstalled([string]$InstallRoot, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + $GoToolsetPath = Resolve-Versioned-Installation-Path $InstallRoot $SpecificVersion + $GoBinPath = (Join-Path $GoToolsetPath "bin") + Say-Verbose "Is-ToolsetInstalled: GoToolsetPath=$GoToolsetPath" + # A few basic checks to see if a likely usable toolset is installed. + # If these fail, it will be reinstalled. + return (Test-Path $GoToolsetPath -PathType Container) -and + ( + (Test-Path (Join-Path $GoBinPath "go") -PathType Leaf) -or + (Test-Path (Join-Path $GoBinPath "go.exe") -PathType Leaf) + ) +} + +function Get-Absolute-Path([string]$RelativeOrAbsolutePath) { + # Too much spam + # Say-Invocation $MyInvocation + + return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath) +} + +function Extract-Zip([string]$ArchivePath, [string]$OutPath) { + Say-Invocation $MyInvocation + + Load-Assembly -Assembly System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($ArchivePath, $OutPath) +} + +function Extract-TarGz([string]$ArchivePath, [string]$OutPath) { + Say-Invocation $MyInvocation + + if (-not (Test-Path $OutPath)) { + New-Item -ItemType Directory -Force -Path $OutPath + } + + try { + & tar -C $OutPath -xzf $ArchivePath + if ($LASTEXITCODE -ne 0) { + throw "tar exit code: $LASTEXITCODE" + } + } + catch { + throw "Failed to extract the tar.gz archive `"$ArchivePath`". Error: $_" + } +} + +function Extract-ToolsetArchive([string]$ArchivePath, [string]$InstallRoot, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + $GoRootPath = Resolve-Versioned-Installation-Path $InstallRoot $SpecificVersion + # First extract to a temporary directory to avoid partial extraction to the final location. This + # makes it so rerunning the script fixes a problem in case of an interruption. Don't use + # systemwide temp directory because Move-Item from there has been observed to fail on Linux. + $TempGoExtractDir = Join-Path $InstallRoot ".tmp-extract" + $TempGoRootPath = Resolve-Versioned-Installation-Path $TempGoExtractDir $SpecificVersion + + # Clean up directories from a previous attempt. + if (Test-Path $GoRootPath) { + Remove-Item $GoRootPath -Recurse -Force + } + if (Test-Path $TempGoExtractDir) { + Remove-Item $TempGoExtractDir -Recurse -Force + } + if (Test-Path $TempGoRootPath) { + Remove-Item $TempGoRootPath -Recurse -Force + } + + try { + switch ([System.IO.Path]::GetExtension($ArchivePath).ToLowerInvariant()) { + ".zip" { Extract-Zip $ArchivePath $TempGoRootPath } + ".gz" { Extract-TarGz $ArchivePath $TempGoRootPath } + default { throw "Unsupported archive type: $ArchivePath" } + } + + # Move contents of inner "go" dir to the output path to avoid unwanted extra dir. + Move-Item (Join-Path $TempGoRootPath "go") $GoRootPath + + $GoRootPath = "" + } + finally { + if ($GoRootPath -ne "" -and (Test-Path $GoRootPath)) { + Remove-Item $GoRootPath -Recurse -Force + } + if (Test-Path $TempGoExtractDir) { + Remove-Item $TempGoExtractDir -Recurse -Force + } + if (Test-Path $TempGoRootPath) { + Remove-Item $TempGoRootPath -Recurse -Force + } + } +} + +function DownloadJson([string]$Source) { + $Text = DownloadString $Source + + try { + return ConvertFrom-Json $Text + } + catch { + Say-Verbose "Failed to parse the JSON response from '$Source': $Text" + throw $_ + } +} + +function DownloadString([string]$Source) { + $Stream = $null + $Reader = $null + + # To make sure errors are accurate and useful, attempt to get the target first. This prevents a + # situation where we succesfully download bing.com after a failed redirect, try to parse the + # HTML as JSON, and present a confusing error message. + if ($Source -like "https://aka.ms/*") { + $DirectSource = Get-AkaMSRedirectTarget $Source + if (!$DirectSource) { + throw "Failed to aka.ms redirect for URL: $Source" + } + $Source = $DirectSource + } + + try { + $Response = GetHTTPResponse -Uri $Source + $Stream = $Response.Content.ReadAsStreamAsync().Result + $Reader = New-Object System.IO.StreamReader($Stream) + return $Reader.ReadToEnd() + } + finally { + if ($null -ne $Stream) { + $Stream.Dispose() + } + if ($null -ne $Reader) { + $Reader.Dispose() + } + } +} + +function DownloadFile($Source, [string]$OutPath) { + if ($Source -notlike "http*") { + # Using System.IO.Path.GetFullPath to get the current directory + # does not work in this context - $pwd gives the current directory + if (![System.IO.Path]::IsPathRooted($Source)) { + $Source = $(Join-Path -Path $pwd -ChildPath $Source) + } + $Source = Get-Absolute-Path $Source + Say "Copying file from $Source to $OutPath" + Copy-Item $Source $OutPath + return + } + + $Stream = $null + + try { + $Response = GetHTTPResponse -Uri $Source + $Stream = $Response.Content.ReadAsStreamAsync().Result + $File = [System.IO.File]::Create($OutPath) + $Stream.CopyTo($File) + $File.Close() + + ValidateRemoteLocalFileSizes -LocalFileOutPath $OutPath -SourceUri $Source + } + finally { + if ($null -ne $Stream) { + $Stream.Dispose() + } + } +} + +function ValidateRemoteLocalFileSizes([string]$LocalFileOutPath, $SourceUri) { + try { + $remoteFileSize = Get-Remote-File-Size -zipUri $SourceUri + $fileSize = [long](Get-Item $LocalFileOutPath).Length + Say "Downloaded file $SourceUri size is $fileSize bytes." + + if ((![string]::IsNullOrEmpty($remoteFileSize)) -and !([string]::IsNullOrEmpty($fileSize)) ) { + if ($remoteFileSize -ne $fileSize) { + Say "The remote and local file sizes are not equal. Remote file size is $remoteFileSize bytes and local size is $fileSize bytes. The local package may be corrupted." + } + else { + Say "The remote and local file sizes are equal." + } + } + else { + Say "Either downloaded or local package size can not be measured. One of them may be corrupted." + } + } + catch { + Say "Either downloaded or local package size can not be measured. One of them may be corrupted." + } +} + +function Remove-FileSafely($Path) { + try { + if (Test-Path $Path) { + Remove-Item $Path + Say-Verbose "The temporary file `"$Path`" was removed." + } + else { + Say-Verbose "The temporary file `"$Path`" does not exist, therefore is not removed." + } + } + catch { + Say-Warning "Failed to remove the temporary file: `"$Path`", remove it manually." + } +} + +function Prepend-ToolsetPathEnv([string]$InstallRoot, [string]$SpecificVersion) { + Say-Invocation $MyInvocation + + $GoRootPath = Resolve-Versioned-Installation-Path $InstallRoot $SpecificVersion + $BinPath = Get-Absolute-Path (Join-Path -Path $GoRootPath -ChildPath "bin") + + if (-Not $NoPath) { + $SuffixedBinPath = $BinPath + [System.IO.Path]::PathSeparator + if (-Not $env:PATH.Contains($SuffixedBinPath)) { + Say "Adding to current process PATH: $BinPath" + Say "Note: This change will not be visible if PowerShell was run as a child process." + $env:PATH = $SuffixedBinPath + $env:PATH + Say-Verbose "The current process PATH is now `"$env:PATH`"." + } + else { + Say "Current process PATH already contains `"$BinPath`"" + } + } + else { + Say "Binaries can be found in $BinPath" + } + + if ($AzurePipelinePath) { + Say "Running an Azure Pipelines logging command to prepend `"$BinPath`" to the PATH." + Say "##vso[task.prependpath]$BinPath" + } +} + +function PrintDryRunOutput($Invocation) { + $RepeatableCommand = ".\$ScriptName -Version `"go$SpecificVersion`" -InstallDir `"$InstallRoot`" -OS `"$CLIOS`" -Architecture `"$CLIArchitecture`"" + + foreach ($key in $Invocation.BoundParameters.Keys) { + if (-not (@("Version","InstallDir","OS","Architecture","DryRun") -contains $key)) { + $RepeatableCommand+=" -$key `"$($Invocation.BoundParameters[$key])`"" + } + } + Say "Repeatable invocation: $RepeatableCommand" +} + +function Get-AkaMSRedirectTarget([string] $akaMsLink) { + $akaMsDownloadLink=$null + + for ($maxRedirections = 9; $maxRedirections -ge 0; $maxRedirections--) + { + #get HTTP response + #do not pass credentials as a part of the $akaMsLink and do not apply credentials in the GetHTTPResponse function + #otherwise the redirect link would have credentials as well + #it would result in applying credentials twice to the resulting link and thus breaking it, and in echoing credentials to the output as a part of redirect link + $Response= GetHTTPResponse -Uri $akaMsLink -HeaderOnly $true -DisableRedirect $true -DisableFeedCredential $true + Say-Verbose "Received response:`n$Response" + + if ([string]::IsNullOrEmpty($Response)) { + Say-Verbose "The link '$akaMsLink' is not valid: failed to get redirect location. The resource is not available." + return $null + } + + #if HTTP code is 301 (Moved Permanently), the redirect link exists + if ($Response.StatusCode -eq 301) + { + try { + $akaMsDownloadLink = $Response.Headers.GetValues("Location")[0] + + if ([string]::IsNullOrEmpty($akaMsDownloadLink)) { + Say-Verbose "The link '$akaMsLink' is not valid: server returned 301 (Moved Permanently), but the headers do not contain the redirect location." + return $null + } + + Say-Verbose "The redirect location retrieved: '$akaMsDownloadLink'." + # This may yet be a link to another redirection. Attempt to retrieve the page again. + $akaMsLink = $akaMsDownloadLink + continue + } + catch { + Say-Verbose "The link '$akaMsLink' is not valid: failed to get redirect location." + return $null + } + } + elseif ((($Response.StatusCode -lt 300) -or ($Response.StatusCode -ge 400)) -and (-not [string]::IsNullOrEmpty($akaMsDownloadLink))) + { + # Redirections have ended. + return $akaMsDownloadLink + } + + Say-Verbose "The link '$akaMsLink' is not valid: failed to retrieve the redirection location." + return $null + } + + Say-Verbose "Aka.ms links have redirected more than the maximum allowed redirections. This may be caused by a cyclic redirection of aka.ms links." + return $null +} + +# Strict mode means attempting to access a JSON key that doesn't exist fails harshly. +# This utility helps make JSON access a bit more concise under those rules. +# https://github.com/PowerShell/PowerShell/issues/10875 +function Get-OrNull($Target, $Property) { + if ($Target -and $Target.PSObject.Properties[$Property]) { + return $Target.PSObject.Properties[$Property].Value + } + return $null +} + +function Get-AssetInformation([string]$NormalizedVersion, [string]$OS, [string]$Architecture) { + Say-Invocation $MyInvocation + + #construct aka.ms link like "https://aka.ms/golang/release/latest/go1.23.assets.json" + $AkaMsLink = "https://aka.ms/golang/release/latest" + $AkaMsLink +="/$NormalizedVersion.assets.json" + Say-Verbose "Constructed assets.json aka.ms link: '$AkaMsLink'." + + $Assets = DownloadJson $AkaMsLink + $MatchingArches = @($Assets.arches | Where-Object { + $Env = Get-OrNull $_ 'env' + return (Get-OrNull $Env 'GOOS') -eq $OS -and + (Get-OrNull $Env 'GOARCH') -eq $Architecture + }) + + foreach ($arch in $MatchingArches) { + Say-Verbose "Matching env '$($arch.env)'." + } + + if ($MatchingArches.Count -ne 1) { + throw "Failed to find exactly one matching asset for OS '$OS' and architecture '$Architecture'." + } + + return ($MatchingArches[0], $Assets.version) +} + +function Prepare-Install-Directory { + New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null +} + +# The following marker is used by microsoft/go-infra tests to insert more logic that runs before any +# installation happens and may stop the script before installation. This allows unit testing without +# adding additional inputs and complexity only used by tests. + +# [END OF FUNCTIONS] + +if ($Help) { + Get-Help $PSCommandPath -Examples + exit +} + +Say "Microsoft Go Install Script version $MicrosoftGoInstallScriptVersion" + +Say-Verbose "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:" +Say-Verbose "- The toolset needs to be installed without user interaction and without admin rights." +Say-Verbose "- The toolset installation doesn't need to persist across multiple CI runs." +Say-Verbose "Visit https://github.com/microsoft/go for a list of other ways to install Microsoft Go.`r`n" + +Measure-Action "Product discovery" { + $script:CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture + $script:CLIOS = Get-CLIOS-From-OS $OS + $script:NormalizedVersion = Get-NormalizedVersion $Version + Say-Verbose "Normalized version: '$NormalizedVersion'" +} + +if ($ArchivePath -eq "") { + $ArchivePath = Get-GeneratedArchivePath $CLIOS + Say-Verbose "Generated archive path: $ArchivePath" +} + +$InstallRoot = Resolve-Installation-Path $InstallDir +Say-Verbose "InstallRoot: $InstallRoot" + +$ScriptName = $MyInvocation.MyCommand.Name + +Say "Fetching information for version '$Version'." +($Arch, $SpecificVersion) = Get-AssetInformation $NormalizedVersion $CLIOS $CLIArchitecture + +$DownloadLink = $Arch.url +Say-Verbose "Found download link $DownloadLink with version $SpecificVersion" + +if (-Not $DryRun) { + Say-Verbose "Checking if the version $SpecificVersion is already installed" + if (Is-ToolsetInstalled -InstallRoot $InstallRoot -SpecificVersion $SpecificVersion) { + Say "Microsoft Go version '$SpecificVersion' is already installed." + Measure-Action "Setting up shell environment" { Prepend-ToolsetPathEnv -InstallRoot $InstallRoot -SpecificVersion $SpecificVersion } + return + } +} + +if ($DryRun) { + PrintDryRunOutput $MyInvocation + return +} + +Measure-Action "Installation directory preparation" { Prepare-Install-Directory } + +Say-Verbose "Zip path: $ArchivePath" + +Say-Verbose "Downloading link $DownloadLink" + +try { + Measure-Action "Package download" { DownloadFile -Source $DownloadLink -OutPath $ArchivePath } + Say-Verbose "Download succeeded." +} +catch { + $StatusCode = $null + $ErrorMessage = $null + + if ($PSItem.Exception.Data.Contains("StatusCode")) { + $StatusCode = $PSItem.Exception.Data["StatusCode"] + } + + if ($PSItem.Exception.Data.Contains("ErrorMessage")) { + $ErrorMessage = $PSItem.Exception.Data["ErrorMessage"] + } else { + $ErrorMessage = $PSItem.Exception.Message + } + + if (-not $KeepArchive) { + Remove-FileSafely -Path $ArchivePath + } + + throw "Downloading has failed with error:`nUri: $DownloadLink`nStatusCode: $StatusCode`nError: $ErrorMessage" +} + +Say "Extracting the archive." +Measure-Action "Archive extraction" { Extract-ToolsetArchive -ArchivePath $ArchivePath -InstallRoot $InstallRoot -SpecificVersion $SpecificVersion } + +Say-Verbose "Checking installation: version = $SpecificVersion" +$isAssetInstalled = Is-ToolsetInstalled -InstallRoot $InstallRoot -SpecificVersion $SpecificVersion + +# Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm. +if (!$isAssetInstalled) { + Say-Error "Failed to verify that the toolset was installed.`nInstallation source: $DownloadLink.`nInstallation location: $InstallRoot.`nReport the bug at https://github.com/microsoft/go/issues." + throw "Toolset with version $SpecificVersion failed to install with an unknown error." +} + +if (-not $KeepArchive) { + Remove-FileSafely -Path $ArchivePath +} + +Measure-Action "Setting up environment PATH to find 'go' command" { Prepend-ToolsetPathEnv -InstallRoot $InstallRoot -SpecificVersion $SpecificVersion } + +Say "Installed version is $SpecificVersion" +Say "Installation finished" diff --git a/eng/_util/go.mod b/eng/_util/go.mod index 1908c85092..897bab5c0f 100644 --- a/eng/_util/go.mod +++ b/eng/_util/go.mod @@ -4,10 +4,11 @@ module github.com/microsoft/go/_util -go 1.21 +go 1.22.0 require ( github.com/microsoft/go-infra v0.0.5 + github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67 golang.org/x/sys v0.27.0 gotest.tools/gotestsum v1.12.0 ) diff --git a/eng/_util/go.sum b/eng/_util/go.sum index 428da6543f..47684ef2a2 100644 --- a/eng/_util/go.sum +++ b/eng/_util/go.sum @@ -29,6 +29,8 @@ github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 h1:YH424zrwLTlyHS github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5/go.mod h1:PoGiBqKSQK1vIfQ+yVaFcGjDySHvym6FM1cNYnwzbrY= github.com/microsoft/go-infra v0.0.5 h1:2vfRe5hJUwJvnXcHDJiv/gqE0XIqCHnwXBFxXRm5+SI= github.com/microsoft/go-infra v0.0.5/go.mod h1:abvc0FBd6VZIdqeJEgo3+SDIE3wSbjpvPhRZ0i21pls= +github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67 h1:i/EXV3vFmJ6UYZ8a4fM2ssKlvSstxvJfiUHau81iNZE= +github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67/go.mod h1:SFsdKAEHdmGsGoh8FkksVaxoQ3rnnJ/TBqN09Ml/0Cw= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/eng/_util/tools/installscript_test.go b/eng/_util/tools/installscript_test.go new file mode 100644 index 0000000000..18cc6ec90d --- /dev/null +++ b/eng/_util/tools/installscript_test.go @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testutil + +import ( + "os/exec" + "testing" +) + +func TestInstallScriptUpToDate(t *testing.T) { + cmd := exec.Command("go", "run", "github.com/microsoft/go-infra/goinstallscript", "-check") + cmd.Dir = ".." + out, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("goinstallscript is not up to date: %v, %v", string(out), err) + t.Errorf("To update, in eng/_util, run: go run github.com/microsoft/go-infra/goinstallscript") + t.Fail() + } +} diff --git a/eng/_util/tools/tools.go b/eng/_util/tools/tools.go new file mode 100644 index 0000000000..1bb08dabc4 --- /dev/null +++ b/eng/_util/tools/tools.go @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build tools + +package tools + +import ( + _ "github.com/microsoft/go-infra/goinstallscript/powershell" +) diff --git a/eng/pipeline/stages/run-stage.yml b/eng/pipeline/stages/run-stage.yml index 34918364cb..3a5d7af961 100644 --- a/eng/pipeline/stages/run-stage.yml +++ b/eng/pipeline/stages/run-stage.yml @@ -94,6 +94,11 @@ stages: - name: MS_GO_UTIL_ALLOW_ONLY_MINIMAL_DEPS value: '1' + # On darwin (macOS) we haven't released a Microsoft Go toolchain. + - ${{ if eq(parameters.builder.os, 'darwin') }}: + - name: MS_USE_PATH_GO + value: '1' + - ${{ if eq(parameters.builder.config, 'codeql_inner') }}: # Manually specify the repository being scanned by this job because # CodeQL can't detect the inner repository (the submodule) @@ -144,10 +149,16 @@ stages: # Initialize stage 0 toolset ahead of time so we can track timing data separately from the # build operations. When we call this script again later, it won't download Go again. - - pwsh: | - . eng/utilities.ps1 - Get-Stage0GoRoot - displayName: Init stage 0 Go toolset + - ${{ if eq(parameters.builder.os, 'darwin') }}: + - task: GoTool@0 + inputs: + version: '1.23.3' + displayName: Init upstream stage 0 Go toolset + - ${{ else }}: + - pwsh: | + . eng/utilities.ps1 + Download-Stage0 + displayName: Init stage 0 Go toolset - template: ../steps/init-submodule-task.yml diff --git a/eng/run.ps1 b/eng/run.ps1 index 07d3c9fa7d..d3cebeafba 100644 --- a/eng/run.ps1 +++ b/eng/run.ps1 @@ -15,9 +15,12 @@ For example, to build the repository: To list all possible tools: run.ps1 -Builds 'eng/_util/cmd//.go' and runs it using the list of -arguments. If necessary, this command automatically installs Go and downloads -the dependencies of the tool. +Builds 'eng/_util/cmd//.go' and runs it using the list of arguments. + +This command automatically installs a known version of Microsoft Go that will be +used to build the tools. The known version of Go will also be used to build the +Go source code, if it's built. Set environment variable "MS_USE_PATH_GO" to 1 to +your own Go from PATH instead. Every tool accepts a '-h' argument to show tool usage help. #> @@ -79,8 +82,20 @@ if (-not ($tool_source -is [System.IO.FileInfo])) { # Now that we have a single result, navigate upwards to see which module it's in. $tool_module = $tool_source.Directory.Parent.Parent.FullName -# Get (downloading if necessary) the GOROOT directory of a stage 0 Go. -$stage0_goroot = Get-Stage0GoRoot +# Download a consistent stage 0 version of Go unless opted out. +if ($env:MS_USE_PATH_GO -eq "1") { + try { + Write-Host "Using $(go version) from '$(go env GOROOT)'. Results may differ from CI environment." + } catch { + Write-Host "Error: 'go' is most likely not in PATH. To download the known version, set 'MS_USE_PATH_GO' to '0' or unset it, then try again." + Write-Host "Exception: $_" + exit 1 + } +} else { + Download-Stage0 +} + +$stage0_goroot = & go env GOROOT # The tool may need to know where our copy of Go is located. Save it in env to give it access. Don't # pass it to the tool as an arg, becuase that would complicate arg handling in each tool. diff --git a/eng/utilities.ps1 b/eng/utilities.ps1 index 784074727e..a6929ec1eb 100644 --- a/eng/utilities.ps1 +++ b/eng/utilities.ps1 @@ -19,103 +19,16 @@ if ($host.Version.Major -lt 6) { throw "Missing prerequisites; see logs above for details." } -function Get-Stage0GoRoot() { +function Download-Stage0() { # We need Go installed in order to build Go, but our common build environment doesn't have it - # pre-installed. This CI script installs a consistent, official version of Go to a directory in - # $HOME to handle this. This also makes it easier to locally repro issues in CI that involve a + # pre-installed (or the right version pre-installed). This CI script installs a consistent version + # of Go to handle this. This also makes it easier to locally repro issues in CI that involve a # specific version of Go. The downloaded copy of Go is called the "stage 0" version. - $stage0_go_version = '1.22.6' + $stage0_go_version = 'go1.23.3-1' - $proc_arch = ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture).ToString().ToLowerInvariant() - if ($IsWindows) { - switch ($proc_arch) { - 'x64' { - $stage0_go_sha256 = '6023083a6e4d3199b44c37e9ba7b25d9674da20fd846a35ee5f9589d81c21a6a' - $stage0_go_suffix = 'windows-amd64.zip' - } - 'arm64' { - $stage0_go_sha256 = '7cf55f357ba8116cd3bff992980e20a704ba451b3dab341cf1787b133d900512' - $stage0_go_suffix = 'windows-arm64.zip' - } - Default { throw "Unable to match Windows '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." } - } - } elseif ($IsLinux) { - switch ($proc_arch) { - 'x64' { - $stage0_go_sha256 = '999805bed7d9039ec3da1a53bfbcafc13e367da52aa823cb60b68ba22d44c616' - $stage0_go_suffix = 'linux-amd64.tar.gz' - } - 'arm64' { - $stage0_go_sha256 = 'c15fa895341b8eaf7f219fada25c36a610eb042985dc1a912410c1c90098eaf2' - $stage0_go_suffix = 'linux-arm64.tar.gz' - } - Default { throw "Unable to match Linux '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." } - } - } elseif ($IsMacOS) { - switch ($proc_arch) { - 'x64' { - $stage0_go_sha256 = '9c3c0124b01b5365f73a1489649f78f971ecf84844ad9ca58fde133096ddb61b' - $stage0_go_suffix = 'darwin-amd64.tar.gz' - } - 'arm64' { - $stage0_go_sha256 = 'ebac39fd44fc22feed1bb519af431c84c55776e39b30f4fd62930da9c0cfd1e3' - $stage0_go_suffix = 'darwin-arm64.tar.gz' - } - Default { throw "Unable to match macOS '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." } - } - } else { - throw "Current OS/Platform is not supported by the Microsoft scripts to build Go." - } - $stage0_url = "https://golang.org/dl/go${stage0_go_version}.${stage0_go_suffix}" - - # Ideally we could set up stage 0 inside the repository, rather than - # userprofile. Tracked by: https://github.com/microsoft/go/issues/12 - $stage0_dir = Join-Path $HOME ".go-stage-0" $stage0_go_version - - # A file that indicates that this version of the stage 0 Go toolset has already been installed. - $download_complete_indicator = Join-Path $stage0_dir ".downloaded-$stage0_go_sha256" - - if (-not (Test-Path $download_complete_indicator -PathType Leaf)) { - Write-Host "Downloading stage 0 Go compiler and extracting to '$stage0_dir' ..." - - # Clear existing stage0 dir in case it's in a broken state. - Remove-Item -Recurse -Force $stage0_dir -ErrorAction Ignore - New-Item -ItemType Directory $stage0_dir | Out-Null - - $go_tarball = Join-Path $stage0_dir "go.$stage0_go_suffix" - - Write-Host "Downloading from '$stage0_url' to '$go_tarball'..." - Invoke-WithRetry -MaxAttempts 5 { - (New-Object System.Net.WebClient).DownloadFile($stage0_url, $go_tarball) - } - - Write-Host "Comparing checksum..." - $actual_hash = (Get-FileHash $go_tarball -Algorithm SHA256).Hash.ToLowerInvariant() - if ($actual_hash -ne $stage0_go_sha256) { - Write-Host "" - Write-Host "Error: hash of downloaded file '$go_tarball' doesn't match expected value:" - Write-Host "Actual: $actual_hash" - Write-Host "Expected: $stage0_go_sha256" - Write-Host "Visit https://golang.org/dl/ to see the list of expected hashes." - - throw "Checksum mismatch. See logs above for details." - } - - Write-Host "Extracting '$go_tarball' to '$stage0_dir'..." - if ($go_tarball.EndsWith(".zip")) { - Extract-Zip $go_tarball $stage0_dir - } elseif ($go_tarball.EndsWith(".tar.gz")) { - Extract-TarGz $go_tarball $stage0_dir - } - Remove-Item "$go_tarball" - - New-Item -ItemType File "$download_complete_indicator" | Out-Null - - Write-Host "Done extracting stage 0 Go compiler to '$stage0_dir'" - } - - # Return GOROOT: contains "bin/go". - return Join-Path $stage0_dir "go" + # Source the install script so that we can use the PATH it assigns. + $installScriptPath = Join-Path $PSScriptRoot "_util" "go-install.ps1" + . $installScriptPath -Version $stage0_go_version } # Copied from https://github.com/dotnet/install-scripts/blob/49d5da7f7d313aa65d24fe95cc29767faef553fd/src/dotnet-install.ps1#L180-L197