Use Microsoft Go as stage 0

eng/_util/go.mod

@@ -4,10 +4,11 @@
 
 module github.com/microsoft/go/_util
 
-go 1.21
+go 1.22.0
 
 require (
 	github.com/microsoft/go-infra v0.0.5
+	github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67
 	golang.org/x/sys v0.27.0
 	gotest.tools/gotestsum v1.12.0
 )

eng/_util/go.sum

@@ -29,6 +29,8 @@ github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 h1:YH424zrwLTlyHS
 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5/go.mod h1:PoGiBqKSQK1vIfQ+yVaFcGjDySHvym6FM1cNYnwzbrY=
 github.com/microsoft/go-infra v0.0.5 h1:2vfRe5hJUwJvnXcHDJiv/gqE0XIqCHnwXBFxXRm5+SI=
 github.com/microsoft/go-infra v0.0.5/go.mod h1:abvc0FBd6VZIdqeJEgo3+SDIE3wSbjpvPhRZ0i21pls=
+github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67 h1:i/EXV3vFmJ6UYZ8a4fM2ssKlvSstxvJfiUHau81iNZE=
+github.com/microsoft/go-infra/goinstallscript v0.0.0-20241113173623-26aea3823c67/go.mod h1:SFsdKAEHdmGsGoh8FkksVaxoQ3rnnJ/TBqN09Ml/0Cw=
 github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
 github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

eng/_util/tools/installscript_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testutil
+
+import (
+	"os/exec"
+	"testing"
+)
+
+func TestInstallScriptUpToDate(t *testing.T) {
+	cmd := exec.Command("go", "run", "github.com/microsoft/go-infra/goinstallscript", "-check")
+	cmd.Dir = ".."
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("goinstallscript is not up to date: %v, %v", string(out), err)
+		t.Errorf("To update, in eng/_util, run: go run github.com/microsoft/go-infra/goinstallscript")
+		t.Fail()
+	}
+}

eng/_util/tools/tools.go

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build tools
+
+package tools
+
+import (
+	_ "github.com/microsoft/go-infra/goinstallscript/powershell"
+)

eng/pipeline/stages/run-stage.yml

@@ -94,6 +94,11 @@ stages:
             - name: MS_GO_UTIL_ALLOW_ONLY_MINIMAL_DEPS
               value: '1'
 
+          # On darwin (macOS) we haven't released a Microsoft Go toolchain.
+          - ${{ if eq(parameters.builder.os, 'darwin') }}:
+            - name: MS_USE_PATH_GO
+              value: '1'
+
           - ${{ if eq(parameters.builder.config, 'codeql_inner') }}:
             # Manually specify the repository being scanned by this job because
             # CodeQL can't detect the inner repository (the submodule)
@@ -142,12 +147,19 @@ stages:
                 Write-Host "##vso[task.setvariable variable=GO_MAKE_MAX_RETRY_ATTEMPTS]5"
               displayName: Increase 'make' retry attempts
 
-          # Initialize stage 0 toolset ahead of time so we can track timing data separately from the
-          # build operations. When we call this script again later, it won't download Go again.
-          - pwsh: |
-              . eng/utilities.ps1
-              Get-Stage0GoRoot
-            displayName: Init stage 0 Go toolset
+          - ${{ if eq(parameters.builder.os, 'darwin') }}:
+            # Initialize stage 0 toolset ahead of time so we can track timing data separately from the
+            # build operations. When we call this script again later, it won't download Go again.
+            - pwsh: |
+                . eng/utilities.ps1
+                Download-Stage0
+              displayName: Init stage 0 Go toolset
+          - ${{ else }}:
+            # Get upstream stage 0 toolset.
+            - task: GoTool@0
+              inputs:
+                version: '1.23.3'
+              displayName: Init upstream stage 0 Go toolset
 
           - template: ../steps/init-submodule-task.yml
 

eng/run.ps1

@@ -15,9 +15,12 @@ For example, to build the repository:
 To list all possible tools:
   run.ps1
 
-Builds 'eng/_util/cmd/<tool>/<tool>.go' and runs it using the list of
-arguments. If necessary, this command automatically installs Go and downloads
-the dependencies of the tool.
+Builds 'eng/_util/cmd/<tool>/<tool>.go' and runs it using the list of arguments.
+
+This command automatically installs a known version of Microsoft Go that will be
+used to build the tools. The known version of Go will also be used to build the
+Go source code, if it's built. Set environment variable "MS_USE_PATH_GO" to 1 to
+your own Go from PATH instead.
 
 Every tool accepts a '-h' argument to show tool usage help.
 #>
@@ -79,8 +82,20 @@ if (-not ($tool_source -is [System.IO.FileInfo])) {
 # Now that we have a single result, navigate upwards to see which module it's in.
 $tool_module = $tool_source.Directory.Parent.Parent.FullName
 
-# Get (downloading if necessary) the GOROOT directory of a stage 0 Go.
-$stage0_goroot = Get-Stage0GoRoot
+# Download a consistent stage 0 version of Go unless opted out.
+if ($env:MS_USE_PATH_GO -eq "1") {
+  try {
+    Write-Host "Using $(go version) from '$(go env GOROOT)'. Results may differ from CI environment."
+  } catch {
+    Write-Host "Error: 'go' is most likely not in PATH. To download the known version, set 'MS_USE_PATH_GO' to '0' or unset it, then try again."
+    Write-Host "Exception: $_"
+    exit 1
+  }
+} else {
+  Download-Stage0
+}
+
+$stage0_goroot = & go env GOROOT
 
 # The tool may need to know where our copy of Go is located. Save it in env to give it access. Don't
 # pass it to the tool as an arg, becuase that would complicate arg handling in each tool.

eng/utilities.ps1

@@ -19,103 +19,16 @@ if ($host.Version.Major -lt 6) {
   throw "Missing prerequisites; see logs above for details."
 }
 
-function Get-Stage0GoRoot() {
+function Download-Stage0() {
   # We need Go installed in order to build Go, but our common build environment doesn't have it
-  # pre-installed. This CI script installs a consistent, official version of Go to a directory in
-  # $HOME to handle this. This also makes it easier to locally repro issues in CI that involve a
+  # pre-installed (or the right version pre-installed). This CI script installs a consistent version
+  # of Go to handle this. This also makes it easier to locally repro issues in CI that involve a
   # specific version of Go. The downloaded copy of Go is called the "stage 0" version.
-  $stage0_go_version = '1.22.6'
+  $stage0_go_version = 'go1.23.3-1'
 
-  $proc_arch = ([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture).ToString().ToLowerInvariant()
-  if ($IsWindows) {
-    switch ($proc_arch) {
-      'x64' {
-        $stage0_go_sha256 = '6023083a6e4d3199b44c37e9ba7b25d9674da20fd846a35ee5f9589d81c21a6a'
-        $stage0_go_suffix = 'windows-amd64.zip'
-      }
-      'arm64' {
-        $stage0_go_sha256 = '7cf55f357ba8116cd3bff992980e20a704ba451b3dab341cf1787b133d900512'
-        $stage0_go_suffix = 'windows-arm64.zip'
-      }
-      Default { throw "Unable to match Windows '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." }
-    }
-  } elseif ($IsLinux) {
-    switch ($proc_arch) {
-      'x64' {
-        $stage0_go_sha256 = '999805bed7d9039ec3da1a53bfbcafc13e367da52aa823cb60b68ba22d44c616'
-        $stage0_go_suffix = 'linux-amd64.tar.gz'
-      }
-      'arm64' {
-        $stage0_go_sha256 = 'c15fa895341b8eaf7f219fada25c36a610eb042985dc1a912410c1c90098eaf2'
-        $stage0_go_suffix = 'linux-arm64.tar.gz'
-      }
-      Default { throw "Unable to match Linux '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." }
-    }
-  } elseif ($IsMacOS) {
-    switch ($proc_arch) {
-      'x64' {
-        $stage0_go_sha256 = '9c3c0124b01b5365f73a1489649f78f971ecf84844ad9ca58fde133096ddb61b'
-        $stage0_go_suffix = 'darwin-amd64.tar.gz'
-      }
-      'arm64' {
-        $stage0_go_sha256 = 'ebac39fd44fc22feed1bb519af431c84c55776e39b30f4fd62930da9c0cfd1e3'
-        $stage0_go_suffix = 'darwin-arm64.tar.gz'
-      }
-      Default { throw "Unable to match macOS '$proc_arch' to an architecture supported by the Microsoft scripts to build Go." }
-    }
-  } else {
-    throw "Current OS/Platform is not supported by the Microsoft scripts to build Go."
-  }
-  $stage0_url = "https://golang.org/dl/go${stage0_go_version}.${stage0_go_suffix}"
-
-  # Ideally we could set up stage 0 inside the repository, rather than
-  # userprofile. Tracked by: https://github.com/microsoft/go/issues/12
-  $stage0_dir = Join-Path $HOME ".go-stage-0" $stage0_go_version
-
-  # A file that indicates that this version of the stage 0 Go toolset has already been installed.
-  $download_complete_indicator = Join-Path $stage0_dir ".downloaded-$stage0_go_sha256"
-
-  if (-not (Test-Path $download_complete_indicator -PathType Leaf)) {
-    Write-Host "Downloading stage 0 Go compiler and extracting to '$stage0_dir' ..."
-
-    # Clear existing stage0 dir in case it's in a broken state.
-    Remove-Item -Recurse -Force $stage0_dir -ErrorAction Ignore 
-    New-Item -ItemType Directory $stage0_dir | Out-Null
-
-    $go_tarball = Join-Path $stage0_dir "go.$stage0_go_suffix"
-
-    Write-Host "Downloading from '$stage0_url' to '$go_tarball'..."
-    Invoke-WithRetry -MaxAttempts 5 {
-      (New-Object System.Net.WebClient).DownloadFile($stage0_url, $go_tarball)
-    }
-
-    Write-Host "Comparing checksum..."
-    $actual_hash = (Get-FileHash $go_tarball -Algorithm SHA256).Hash.ToLowerInvariant()
-    if ($actual_hash -ne $stage0_go_sha256) {
-      Write-Host ""
-      Write-Host "Error: hash of downloaded file '$go_tarball' doesn't match expected value:"
-      Write-Host "Actual:   $actual_hash"
-      Write-Host "Expected: $stage0_go_sha256"
-      Write-Host "Visit https://golang.org/dl/ to see the list of expected hashes."
-
-      throw "Checksum mismatch. See logs above for details."
-    }
-
-    Write-Host "Extracting '$go_tarball' to '$stage0_dir'..."
-    if ($go_tarball.EndsWith(".zip")) {
-      Extract-Zip $go_tarball $stage0_dir
-    } elseif ($go_tarball.EndsWith(".tar.gz")) {
-      Extract-TarGz $go_tarball $stage0_dir
-    }
-    Remove-Item "$go_tarball"
-
-    New-Item -ItemType File "$download_complete_indicator" | Out-Null
-
-    Write-Host "Done extracting stage 0 Go compiler to '$stage0_dir'"
-  }
-
-  # Return GOROOT: contains "bin/go".
-  return Join-Path $stage0_dir "go"
+  # Source the install script so that we can use the PATH it assigns.
+  $installScriptPath = Join-Path $PSScriptRoot "_util" "go-install.ps1"
+  . $installScriptPath -Version $stage0_go_version
 }
 
 # Copied from https://github.com/dotnet/install-scripts/blob/49d5da7f7d313aa65d24fe95cc29767faef553fd/src/dotnet-install.ps1#L180-L197