From 38d62e36b312e3ba024d29f4f4b887b6eec83992 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Thu, 7 Nov 2024 11:00:37 -0800 Subject: [PATCH] Add macOS buildandpack with signing (#1388) * Add macOS buildandpack with signing Add macOS builds. Rework signing to support macOS hardening and notarization. * Fail if archive contains non-local path * firstError -> cmp.Or * Lowercase filename check --- eng/_util/cmd/sign/README.md | 50 ++ eng/_util/cmd/sign/archive.go | 467 ++++++++++++++++++ eng/_util/cmd/sign/archiveutil.go | 134 +++++ eng/_util/cmd/sign/sign.go | 291 +++++++++++ .../cmd/write-checksum/write-checksum.go | 31 +- eng/_util/internal/checksum/checksum.go | 32 ++ eng/pipeline/rolling-internal-pipeline.yml | 6 + eng/pipeline/stages/builders-to-stages.yml | 34 +- .../stages/go-builder-matrix-stages.yml | 6 + eng/pipeline/stages/pool-1.yml | 4 +- eng/pipeline/stages/pool-2.yml | 5 + eng/pipeline/stages/sign-stage.yml | 95 ++-- eng/signing/.gitignore | 1 + eng/signing/NuGet.config | 2 +- eng/signing/README.md | 31 +- eng/signing/Sign.csproj | 28 ++ eng/signing/Sign.proj | 93 ---- 17 files changed, 1110 insertions(+), 200 deletions(-) create mode 100644 eng/_util/cmd/sign/README.md create mode 100644 eng/_util/cmd/sign/archive.go create mode 100644 eng/_util/cmd/sign/archiveutil.go create mode 100644 eng/_util/cmd/sign/sign.go create mode 100644 eng/_util/internal/checksum/checksum.go create mode 100644 eng/signing/Sign.csproj delete mode 100644 eng/signing/Sign.proj diff --git a/eng/_util/cmd/sign/README.md b/eng/_util/cmd/sign/README.md new file mode 100644 index 00000000000..a74a2beffdd --- /dev/null +++ b/eng/_util/cmd/sign/README.md @@ -0,0 +1,50 @@ +# `sign` and the Microsoft Go signing infrastructure + +Most of the logic for signing (extracting files, repackaging, creating checksums) is implemented by this `sign` command. + +The [`/eng/signing`](/eng/signing) directory contains the MSBuild project that `sign` invokes to run real signing. +The MSBuild project uses [MicroBuild Signing](https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing) (internal Microsoft wiki link). + +To see signing in action, go to [`/eng/pipeline/README.md`](/eng/pipeline/README.md) and follow the link for `microsoft-go`. + +## Dry run + +1. Create the directory `/eng/signing/tosign` and add the `.tar.gz` and `.zip` artifacts to sign. + * Download artifacts from the `microsoft-go` pipeline, for example. + * It's ok to skip downloading some artifacts. The signing process doesn't require all platforms to be present. + * If you specify `-files`, you can use your own directory. +1. From the root of the repository, run `pwsh eng/run.ps1 sign -n` + +The `-n` argument makes it a dry run: it extracts/repacks files in the same way it would if it were signing them, but no signing is done. +This doesn't involve .NET/MSBuild, so this is a good way for a developer to test changes to the signing logic. + +See `pwsh eng/run.ps1 sign -h` for more options. + +## Test signing + +> [!NOTE] +> Test signing has not been observed to work. +> It has been documented for completeness, in case someone wants to try. + +### Prerequisites + +* Windows +* .NET Core SDK 8.0 or later. + * [Download](https://dot.net/download) +* The signing plugin. + 1. Download the latest NuGet Package: https://devdiv.visualstudio.com/DevDiv/_artifacts/feed/MicroBuildToolset/NuGet/MicroBuild.Plugins.Signing + 1. Extract its contents (the file is a zip) to `%userprofile%\.nuget\packages\microbuild.plugins.signing\1.1.900`. + * Optionally make the versioned dir's name match the version of the package you downloaded. It will be discovered dynamically, as a plugin, whether or not the version matches. + +### Test signing run + +1. Set up `tosign` as described in the dry run section. +1. From the root of the repository, run `pwsh eng/run.ps1 sign` + +## Real signing + +This can't be done from a dev machine. +It occurs in the `microsoft-go` pipeline, on a Windows machine. +See [`/eng/pipeline/README.md`](/eng/pipeline/README.md). + +The invocation of `sign` can be found in [`/eng/pipeline/stages/sign-stage.yml`](/eng/pipeline/stages/sign-stage.yml). diff --git a/eng/_util/cmd/sign/archive.go b/eng/_util/cmd/sign/archive.go new file mode 100644 index 00000000000..701e83103eb --- /dev/null +++ b/eng/_util/cmd/sign/archive.go @@ -0,0 +1,467 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "archive/zip" + "cmp" + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" +) + +type archiveType int + +const ( + // zipArchive is a Windows zip archive. + zipArchive archiveType = iota + // tarGzArchive is a macOS or Linux tar.gz archive. + tarGzArchive +) + +type archive struct { + path string + name string + + archiveType archiveType + archiveMacOS bool + + // workDir is a work dir absolute path that is only used for processing this archive. + workDir string + + // repackedPath is a repackaged archive with signed content. Assigned upon completion. + // Windows and macOS archives get repacked. + repackedPath string + // notarizedPath is a repacked archive that has also had the notarization ticket attached. + // Assigned upon completion. + notarizedPath string +} + +func newArchive(p string) (*archive, error) { + name := filepath.Base(p) + a := archive{ + path: p, + name: name, + } + if matchOrPanic("go*.zip", name) { + a.archiveType = zipArchive + } else if matchOrPanic("go*.tar.gz", name) { + a.archiveType = tarGzArchive + } else { + return nil, fmt.Errorf("unknown archive type: %s", p) + } + + if matchOrPanic("go*darwin*.tar.gz", name) { + a.archiveMacOS = true + } + + if err := os.MkdirAll(*tempDir, 0o777); err != nil { + return nil, err + } + workDir, err := os.MkdirTemp(*tempDir, "sign-work-"+name) + if err != nil { + return nil, fmt.Errorf("failed to create work directory: %v", err) + } + workDir, err = filepath.Abs(workDir) + if err != nil { + return nil, err + } + a.workDir = workDir + + return &a, nil +} + +// latestPath returns the path of the file that has the most signing steps applied to it. This +// allows for some generalization across platforms in later steps. +func (a *archive) latestPath() string { + if a.notarizedPath != "" { + return a.notarizedPath + } + if a.repackedPath != "" { + return a.repackedPath + } + return a.path +} + +func (a *archive) sigPath() string { + return filepath.Join(a.workDir, a.name+".sig") +} + +func (a *archive) macHardenPackPath() string { + return filepath.Join(a.workDir, a.name+".ToSignBundle.zip") +} + +func (a *archive) macNotarizePackPath() string { + return filepath.Join(a.workDir, a.name+".ToNotarize.zip") +} + +// entrySignInfo returns signing details for a given file in the Go archive, or nil if the given +// file entry doesn't need to be signed. +func (a *archive) entrySignInfo(name string) *fileToSign { + if a.archiveType == zipArchive { + if strings.HasSuffix(name, ".exe") { + return &fileToSign{ + originalPath: a.path, + fullPath: filepath.Join(a.workDir, "extract", name), + authenticode: "Microsoft400", + } + } + } else if a.archiveMacOS { + if matchOrPanic("go/bin/*", name) || + matchOrPanic("go/pkg/tool/*/*", name) { + + return &fileToSign{ + originalPath: a.path, + zip: true, + } + } + } + return nil +} + +// prepareEntriesToSign extracts files from the archive that need to be signed and returns a list +// of their extracted locations and details about how they should be signed. +func (a *archive) prepareEntriesToSign(ctx context.Context) ([]*fileToSign, error) { + fail := func(err error) ([]*fileToSign, error) { + return nil, fmt.Errorf("failed to extract file from %q: %v", a.path, err) + } + + var results []*fileToSign + + if a.archiveType == zipArchive { + log.Printf("Extracting files to sign from %q", a.path) + zr, err := zip.OpenReader(a.path) + if err != nil { + return fail(err) + } + defer zr.Close() + + if err := eachZipEntry(zr, func(f *zip.File) error { + if err := ctx.Err(); err != nil { + return err + } + if f.FileInfo().IsDir() { + return nil + } + if info := a.entrySignInfo(f.Name); info != nil { + if err := withFileCreate(info.fullPath, func(fWriter *os.File) error { + fReader, err := f.Open() + if err != nil { + return err + } + _, err = io.Copy(fWriter, fReader) + return cmp.Or(err, fReader.Close()) + }); err != nil { + return err + } + results = append(results, info) + } + return nil + }); err != nil { + return fail(err) + } + } else if a.archiveMacOS { + // Store macOS files to sign in a zip. Zipping is needed for this platform specifically, + // and the "Zip=true" feature mentioned in the doc only works when signing on a macOS + // runtime, so we need to do it ourselves. + // https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/19841/Additional-Requirements-for-Signing-or-Notarizing-Mac-Files + fts := &fileToSign{ + originalPath: a.path, + fullPath: a.macHardenPackPath(), + authenticode: "MacDeveloperHarden", + } + log.Printf("Creating macOS file hardening bundle at %q", fts.fullPath) + if err := withZipCreate(fts.fullPath, func(zw *zip.Writer) error { + return a.extractMacOSEntriesToZip(ctx, zw) + }); err != nil { + return fail(err) + } + results = append(results, fts) + } + + return results, nil +} + +func (a *archive) extractMacOSEntriesToZip(ctx context.Context, zw *zip.Writer) error { + // Open tar.gz macOS archive to put files into the zip. + writtenNames := make(map[string]struct{}) + return withTarGzOpen(a.path, func(tr *tar.Reader) error { + return eachTarEntry(tr, func(header *tar.Header, r io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + if header.Typeflag != tar.TypeReg { + return nil + } + if info := a.entrySignInfo(header.Name); info != nil { + if !info.zip { + return fmt.Errorf("unexpected file to sign directly rather than include in the zip batch: %q", header.Name) + } + + base := filepath.Base(header.Name) + if _, ok := writtenNames[base]; ok { + return fmt.Errorf("duplicate file name in archive: %q", base) + } + writtenNames[base] = struct{}{} + + w, err := zw.CreateHeader(&zip.FileHeader{ + Name: base, + }) + if err != nil { + return err + } + _, err = io.Copy(w, r) + return err + } + return nil + }) + }) +} + +func (a *archive) repackSignedEntries(ctx context.Context) error { + targetPath := filepath.Join(a.workDir, a.name+".WithSignedContent") + if a.archiveType == zipArchive { + log.Printf("Repacking signed content to %q", targetPath) + if err := withZipOpen(a.path, func(zr *zip.ReadCloser) error { + return withZipCreate(targetPath, func(zw *zip.Writer) error { + return eachZipEntry(zr, func(f *zip.File) error { + if err := ctx.Err(); err != nil { + return err + } + return a.writeZipRepackEntry(f, zw) + }) + }) + }); err != nil { + return err + } + a.repackedPath = targetPath + } else if a.archiveMacOS { + log.Printf("Repacking hardened content to %q", targetPath) + // Open the original tar.gz for header info and to read unchanged files from. + if err := withTarGzOpen(a.path, func(originalTR *tar.Reader) error { + // Create the new tar.gz that we're assembling. + return withTarGzCreate(targetPath, func(outTW *tar.Writer) error { + // Open the zip payload we got back from the signing service. + return withZipOpen(a.macHardenPackPath(), func(zrc *zip.ReadCloser) error { + // Iterate through the original tar.gz file to populate the target. + return eachTarEntry(originalTR, func(hdr *tar.Header, originalR io.Reader) error { + if err := ctx.Err(); err != nil { + return err + } + return a.writeTarRepackEntry(hdr, originalR, &zrc.Reader, outTW) + }) + }) + }) + }); err != nil { + return err + } + a.repackedPath = targetPath + } + return nil +} + +// writeZipRepackEntry looks at one entry in the original zip and creates a corresponding entry in +// the output zip. Reads signed entry content from the signed file on disk. If the file hasn't been +// signed, the content is read from the original zip. +func (a *archive) writeZipRepackEntry(original *zip.File, out *zip.Writer) error { + w, err := out.CreateHeader(&zip.FileHeader{ + // Copy necessary original file metadata. + Name: original.Name, + Method: original.Method, + Comment: original.Comment, + Modified: original.Modified, + Extra: original.Extra, + }) + if err != nil { + return err + } + var r io.ReadCloser + // If we have a signed version of this file, read from that. + // Otherwise, read from the original. + if info := a.entrySignInfo(original.Name); info != nil { + log.Printf("Replacing with signed version: %q", original.Name) + r, err = os.Open(info.fullPath) + if err != nil { + return err + } + } else { + r, err = original.Open() + if err != nil { + return err + } + } + _, err = io.Copy(w, r) + return cmp.Or(err, r.Close()) +} + +// writeTarRepackEntry looks at one entry in the original tar.gz and creates a corresponding entry +// in the output tar.gz. Reads signed/hardened entry content from signedPack. Otherwise, the entry +// content is copied from the original. +func (a *archive) writeTarRepackEntry(hdr *tar.Header, original io.Reader, signedPack *zip.Reader, out *tar.Writer) error { + // Always start with header info from the original tar.gz even if we're going to replace the + // file content. This means we don't need to worry about lost metadata due to the zip + // round-trip. + newHeader := &tar.Header{ + // Follow tar.Header documented compat guidance by copying over our selection of fields. + Name: hdr.Name, + Linkname: hdr.Linkname, + + Size: hdr.Size, + Mode: hdr.Mode, + Uid: hdr.Uid, + Gid: hdr.Gid, + Uname: hdr.Uname, + Gname: hdr.Gname, + + ModTime: hdr.ModTime, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + } + isFile := hdr.Typeflag == tar.TypeReg + if info := a.entrySignInfo(hdr.Name); info != nil && isFile { + log.Printf("Replacing with signed version: %q", hdr.Name) + replacementFile, err := signedPack.Open(filepath.Base(hdr.Name)) + if err != nil { + return err + } + defer replacementFile.Close() + // Get the file size to prepare to copy. + stat, err := replacementFile.Stat() + if err != nil { + return err + } + newHeader.Size = stat.Size() + original = replacementFile + } + if err := out.WriteHeader(newHeader); err != nil { + return fmt.Errorf( + "failed to write header for %q: %v", + newHeader.Name, err) + } + if isFile { + _, err := io.Copy(out, original) + if err != nil { + return fmt.Errorf("failed to write %q: %v", newHeader.Name, err) + } + } + // Call Flush to make sure our write was correct. We don't technically need to call Flush here + // because the next WriteHeader will confirm that we e.g. wrote the correct number of bytes. + // However, calling Flush ourselves lets us emit an error that mentions the bad filename + // (rather than the next, unrelated filename). + if err := out.Flush(); err != nil { + return fmt.Errorf("failed to flush %q: %v", newHeader.Name, err) + } + return nil +} + +func (a *archive) prepareNotarize(ctx context.Context) ([]*fileToSign, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if !a.archiveMacOS { + return nil, nil + } + + log.Printf("Creating zip containing the macOS tar.gz to notarize at %q", a.macNotarizePackPath()) + if err := withZipCreate(a.macNotarizePackPath(), func(zw *zip.Writer) error { + w, err := zw.CreateHeader(&zip.FileHeader{ + Name: a.name, + }) + if err != nil { + return err + } + return withFileOpen(a.latestPath(), func(f *os.File) error { + _, err := io.Copy(w, f) + return err + }) + }); err != nil { + return nil, err + } + return []*fileToSign{ + { + originalPath: a.path, + fullPath: a.macNotarizePackPath(), + authenticode: "8020", // Can't specify MacNotarize or MacAppName is not detected. + macAppName: "MicrosoftGo", + }, + }, nil +} + +func (a *archive) unpackNotarize(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + + if !a.archiveMacOS { + return nil + } + + a.notarizedPath = filepath.Join(a.workDir, a.name+".notarized") + log.Printf("Unpacking notarized content to %q", a.notarizedPath) + return withZipOpen(a.macNotarizePackPath(), func(zr *zip.ReadCloser) error { + return eachZipEntry(zr, func(f *zip.File) error { + if err := ctx.Err(); err != nil { + return err + } + if f.Name != a.name { + return fmt.Errorf("unexpected file in notarize zip: %q", f.Name) + } + return withFileCreate(a.notarizedPath, func(w *os.File) error { + r, err := f.Open() + if err != nil { + return err + } + _, err = io.Copy(w, r) + return cmp.Or(err, r.Close()) + }) + }) + }) +} + +func (a *archive) prepareArchiveSignatures(ctx context.Context) ([]*fileToSign, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + // Copy the archive file to have .sig suffix, e.g. "tar.gz" to "tar.gz.sig". The signing + // process sends the "tar.gz.sig" file to get a signature, then replaces the "tar.gz.sig" + // file's content in-place with the result. We need to preemptively make a renamed copy of the + // file so we end up with both the original file and sig on the machine. + log.Printf("Copying file for signature generation: %q -> %q", a.latestPath(), a.sigPath()) + if err := copyFile(a.sigPath(), a.latestPath()); err != nil { + return nil, err + } + return []*fileToSign{ + { + originalPath: a.path, + fullPath: a.sigPath(), + authenticode: "LinuxSignManagedLanguageCompiler", + }, + }, nil +} + +func (a *archive) copyToDestination(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // Create destination if it doesn't exist. + if err := os.MkdirAll(*destinationDir, 0o777); err != nil { + return fmt.Errorf("failed to create destination directory: %v", err) + } + + log.Printf("Copying finished files to destination: %q", a.latestPath()) + if err := copyFile(filepath.Join(*destinationDir, a.name), a.latestPath()); err != nil { + return err + } + if err := copyFile(filepath.Join(*destinationDir, a.name+".sig"), a.sigPath()); err != nil { + return err + } + return nil +} diff --git a/eng/_util/cmd/sign/archiveutil.go b/eng/_util/cmd/sign/archiveutil.go new file mode 100644 index 00000000000..091053752c9 --- /dev/null +++ b/eng/_util/cmd/sign/archiveutil.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "archive/tar" + "archive/zip" + "cmp" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path/filepath" +) + +func eachZipEntry(r *zip.ReadCloser, f func(*zip.File) error) error { + for _, file := range r.File { + // Disallow absolute path, "..", etc. + if !filepath.IsLocal(file.Name) { + return fmt.Errorf("zip contains non-local path: %s", file.Name) + } + if err := f(file); err != nil { + return err + } + } + return nil +} + +func eachTarEntry(r *tar.Reader, f func(*tar.Header, io.Reader) error) error { + for { + header, err := r.Next() + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + // Disallow absolute path, "..", etc. + if !filepath.IsLocal(header.Name) { + return fmt.Errorf("tar contains non-local path: %s", header.Name) + } + if err := f(header, r); err != nil { + return err + } + } +} + +func withFileOpen(path string, f func(*os.File) error) error { + file, err := os.Open(path) + if err != nil { + return err + } + return cmp.Or(f(file), file.Close()) +} + +func withZipOpen(path string, f func(*zip.ReadCloser) error) error { + r, err := zip.OpenReader(path) + if err != nil { + return err + } + return cmp.Or(f(r), r.Close()) +} + +func withTarGzOpen(path string, f func(*tar.Reader) error) error { + return withFileOpen(path, func(file *os.File) error { + gz, err := gzip.NewReader(file) + if err != nil { + return err + } + r := tar.NewReader(gz) + return f(r) + }) +} + +func withFileCreate(path string, f func(*os.File) error) error { + if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil { + return err + } + file, err := os.Create(path) + if err != nil { + return err + } + return cmp.Or(f(file), file.Close()) +} + +func withZipCreate(path string, f func(*zip.Writer) error) error { + return withFileCreate(path, func(file *os.File) error { + w := zip.NewWriter(file) + return cmp.Or(f(w), w.Close()) + }) +} + +func withTarGzCreate(path string, f func(*tar.Writer) error) error { + return withFileCreate(path, func(file *os.File) error { + gzw, err := gzip.NewWriterLevel(file, gzip.BestCompression) + if err != nil { + return err + } + tw := tar.NewWriter(gzw) + return cmp.Or(f(tw), tw.Close(), gzw.Close()) + }) +} + +func copyFile(dst, src string) error { + f, err := os.Open(src) + if err != nil { + return err + } + return cmp.Or(copyToFile(dst, f), f.Close()) +} + +func copyToFile(path string, r io.Reader) error { + if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil { + return err + } + f, err := os.Create(path) + if err != nil { + return err + } + _, err = io.Copy(f, r) + return cmp.Or(err, f.Close()) +} + +// matchOrPanic returns whether name matches the pattern glob, or panics if pattern is invalid. +func matchOrPanic(pattern, name string) bool { + ok, err := filepath.Match(pattern, name) + if err != nil { + panic(err) + } + return ok +} diff --git a/eng/_util/cmd/sign/sign.go b/eng/_util/cmd/sign/sign.go new file mode 100644 index 00000000000..07c7a7bd31f --- /dev/null +++ b/eng/_util/cmd/sign/sign.go @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/microsoft/go/_util/internal/checksum" +) + +const description = ` +This command signs build artifacts using MicroBuild. It is used in the Microsoft Go build pipeline. +Use '-n' to test the command locally. + +Signs in multiple passes. Some steps only apply to certain types of archives: + +1. Archive entries. Extracts specific entries from inside each archive, signs, and repacks. +2. Notarize. macOS archives get a notarization ticket attached to the tar.gz. +3. Signatures. Creates sig files for each archive. +4. Locally creates a .sha256 file for each archive. + +See /eng/_util/cmd/sign/README.md for more information. +` + +var ( + filesGlob = flag.String("files", "eng/signing/tosign/*", "Glob of Go archives to sign.") + destinationDir = flag.String("o", "eng/signing/signed", "Directory to store signed files.") + tempDir = flag.String("temp-dir", "eng/signing/signing-temp", "Directory to store temporary files.") + signingCsprojDir = flag.String("signing-csproj-dir", "eng/signing", "Directory containing Sign.csproj and related files.") + + notarize = flag.Bool("notarize", false, "Notarize macOS archives. This is currently not working in the signing service.") + signType = flag.String("sign-type", "test", "Type of signing to perform. Options: test, real.") + + timeout = flag.Duration("timeout", 0, + "Timeout for signing operations. Zero means no timeout. "+ + "Any MSBuild processes launched by this tool are be manually killed. "+ + "If set to a value lower than AzDO pipeline timeout, this helps avoid pipeline breakage when uploading MSBuild outputs.") + dryRun = flag.Bool("n", false, "Dry run: don't run the MSBuild signing tooling at all, even in test mode. This works on non-Windows platforms.") +) + +func main() { + help := flag.Bool("h", false, "Print this help message.") + + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage:\n") + flag.PrintDefaults() + fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description) + } + + flag.Parse() + if *help { + flag.Usage() + return + } + + if err := run(); err != nil { + log.Printf("error: %v", err) + os.Exit(1) + } +} + +func run() error { + // A context for timeout. This timeout is mainly here to make sure child MSBuild processes are + // terminated. There are some ctx.Err() checks sprinkled into the Go code, but canceling + // quickly during the packaging/repackaging work in Go is not currently important: the Go work + // takes an insignificant amount of time compared to the signing service calls in MSBuild. + var ctx context.Context + if *timeout == 0 { + ctx = context.Background() + } else { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(context.Background(), time.Now().Add(*timeout)) + defer cancel() + } + + archives, err := findArchives(ctx, *filesGlob) + if err != nil { + return err + } + + log.Println("Signing individual files extracted from archives") + + individualFilesToSign, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareEntriesToSign(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "1-Individual", individualFilesToSign); err != nil { + return err + } + + for _, a := range archives { + if err := a.repackSignedEntries(ctx); err != nil { + return err + } + } + + if *notarize { + log.Println("Notarizing macOS archives") + + filesToNotarize, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareNotarize(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "2-Notarize", filesToNotarize); err != nil { + return err + } + + for _, a := range archives { + if err := a.unpackNotarize(ctx); err != nil { + return err + } + } + } else { + log.Println("Skipping notarizing macOS archives") + } + + log.Println("Creating signature files") + + signatureFiles, err := flatMapSlice(archives, func(a *archive) ([]*fileToSign, error) { + return a.prepareArchiveSignatures(ctx) + }) + if err != nil { + return err + } + + if err := sign(ctx, "3-Sigs", signatureFiles); err != nil { + return err + } + + log.Println("Copying finished files to destination") + + for _, a := range archives { + if err := a.copyToDestination(ctx); err != nil { + return err + } + } + + log.Println("Generating checksum files") + + for _, a := range archives { + if err := checksum.WriteSHA256ChecksumFile(filepath.Join(*destinationDir, a.name)); err != nil { + return err + } + } + + return nil +} + +func findArchives(ctx context.Context, glob string) ([]*archive, error) { + files, err := filepath.Glob(glob) + if err != nil { + return nil, fmt.Errorf("failed to glob files: %v", err) + } + + archives := make([]*archive, 0, len(files)) + + // Check for duplicate filenames. At the end of signing, we will put all the results in the + // same directory (even if the sources came from different directories), so catching this + // early saves time. + // + // Use lowercase because we sign on a Windows machine with a case-insensitive filesystem. + archiveFilenames := make(map[string]string) + + for _, f := range files { + if err := ctx.Err(); err != nil { + return nil, err + } + // Ignore checksum files: we always generate new ones. + if strings.HasSuffix(f, ".sha256") { + continue + } + + filenameLower := strings.ToLower(filepath.Base(f)) + if existingF, ok := archiveFilenames[filenameLower]; ok { + return nil, fmt.Errorf("duplicate archive %q, already found %q (comparing lowercase filename)", f, existingF) + } + archiveFilenames[filenameLower] = f + + a, err := newArchive(f) + if err != nil { + return nil, fmt.Errorf("failed to process %q: %v", f, err) + } + archives = append(archives, a) + } + + if len(archives) == 0 { + return nil, fmt.Errorf("no archives found to sign matching glob %q", *filesGlob) + } + + return archives, nil +} + +func sign(ctx context.Context, step string, files []*fileToSign) error { + var sb strings.Builder + sb.WriteString("\n") + sb.WriteString(" \n") + for _, f := range files { + f.WriteMSBuildItem(&sb) + } + sb.WriteString(" \n") + sb.WriteString("\n") + + log.Printf("Signing with props file content:\n%s\n", sb.String()) + if *dryRun { + log.Printf("Dry run: skipping signing.") + return nil + } + + if err := os.MkdirAll(*tempDir, 0o777); err != nil { + return err + } + // Get an absolute path to pass to MSBuild, because our working dirs may not be the same. + // MSBuild in general will resolve paths relative to the csproj. + absTemp, err := filepath.Abs(*tempDir) + if err != nil { + return err + } + propsFilePath := filepath.Join(absTemp, "Sign"+step+".props") + if err := os.WriteFile(propsFilePath, []byte(sb.String()), 0o666); err != nil { + return err + } + + cmd := exec.CommandContext( + ctx, + "dotnet", "build", "Sign.csproj", + "/p:SignFilesDir="+absTemp, + "/p:FilesToSignPropsFile="+propsFilePath, + "/t:AfterBuild", + "/p:SignType="+*signType, + "/bl:"+filepath.Join(absTemp, "Sign"+step+".binlog"), + "/v:n", + ) + cmd.Dir = *signingCsprojDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + log.Printf("Running: %v", cmd) + return cmd.Run() +} + +type fileToSign struct { + originalPath string + fullPath string + authenticode string + // This file is part of a zip payload, e.g. for macOS hardening. + zip bool + // macAppName for notarization. + macAppName string +} + +func (f *fileToSign) WriteMSBuildItem(w io.Writer) { + fmt.Fprintf(w, " \n") +} + +// flatMapSlice sequentially maps each element of es to a slice using f and flattens the resulting +// slices. If any call to f returns an error, the error is returned immediately. +func flatMapSlice[E, R any](es []E, f func(E) ([]R, error)) ([]R, error) { + var results []R + for _, e := range es { + rs, err := f(e) + if err != nil { + return nil, err + } + results = append(results, rs...) + } + return results, nil +} diff --git a/eng/_util/cmd/write-checksum/write-checksum.go b/eng/_util/cmd/write-checksum/write-checksum.go index 191ebd0eae6..abd08ee68ff 100644 --- a/eng/_util/cmd/write-checksum/write-checksum.go +++ b/eng/_util/cmd/write-checksum/write-checksum.go @@ -5,14 +5,11 @@ package main import ( - "crypto/sha256" - "encoding/hex" "flag" "fmt" - "io" "log" - "os" - "path/filepath" + + "github.com/microsoft/go/_util/internal/checksum" ) const description = ` @@ -42,30 +39,8 @@ func main() { log.Fatal("No files specified.") } for _, m := range flag.Args() { - if err := writeSHA256ChecksumFile(m); err != nil { + if err := checksum.WriteSHA256ChecksumFile(m); err != nil { log.Fatal(err) } } } - -func writeSHA256ChecksumFile(path string) error { - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - checksum := sha256.New() - if _, err = io.Copy(checksum, file); err != nil { - return err - } - // Write the checksum in a format that "sha256sum -c" can work with. Use the base path of the - // tarball (not full path, not relative path) because then "sha256sum -c" automatically works - // when the file and the checksum file are downloaded to the same directory. - content := fmt.Sprintf("%v %v\n", hex.EncodeToString(checksum.Sum(nil)), filepath.Base(path)) - outputPath := path + ".sha256" - if err := os.WriteFile(outputPath, []byte(content), 0o666); err != nil { - return err - } - fmt.Printf("Wrote checksum file %q with content: %v", outputPath, content) - return nil -} diff --git a/eng/_util/internal/checksum/checksum.go b/eng/_util/internal/checksum/checksum.go new file mode 100644 index 00000000000..c12d2d0a1d9 --- /dev/null +++ b/eng/_util/internal/checksum/checksum.go @@ -0,0 +1,32 @@ +package checksum + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" +) + +func WriteSHA256ChecksumFile(path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + checksum := sha256.New() + if _, err = io.Copy(checksum, file); err != nil { + return err + } + // Write the checksum in a format that "sha256sum -c" can work with. Use the base path of the + // tarball (not full path, not relative path) because then "sha256sum -c" automatically works + // when the file and the checksum file are downloaded to the same directory. + content := fmt.Sprintf("%v %v\n", hex.EncodeToString(checksum.Sum(nil)), filepath.Base(path)) + outputPath := path + ".sha256" + if err := os.WriteFile(outputPath, []byte(content), 0o666); err != nil { + return err + } + fmt.Printf("Wrote checksum file %q with content: %v", outputPath, content) + return nil +} diff --git a/eng/pipeline/rolling-internal-pipeline.yml b/eng/pipeline/rolling-internal-pipeline.yml index 98d4809a802..8f558e8d892 100644 --- a/eng/pipeline/rolling-internal-pipeline.yml +++ b/eng/pipeline/rolling-internal-pipeline.yml @@ -33,6 +33,11 @@ parameters: type: string default: nil + - name: signExistingRunID + displayName: 'For debugging signing: skip building, and instead sign the artifacts from an existing run. Leave "nil" otherwise.' + type: string + default: 'nil' + variables: - template: variables/pool-providers.yml # MicroBuild configuration. @@ -71,6 +76,7 @@ extends: buildandpack: true official: true sign: true + signExistingRunID: ${{ parameters.signExistingRunID }} createSourceArchive: true createSymbols: true publish: true diff --git a/eng/pipeline/stages/builders-to-stages.yml b/eng/pipeline/stages/builders-to-stages.yml index f5d29d3ede2..84943fc740b 100644 --- a/eng/pipeline/stages/builders-to-stages.yml +++ b/eng/pipeline/stages/builders-to-stages.yml @@ -9,6 +9,8 @@ parameters: builders: [] # If true, include a signing stage+job that depends on all 'buildandpack' builder jobs finishing. sign: false + # If changed to specify an existing pipeline run, skip build and sign the existing run. + signExistingRunID: 'nil' # If true, publish build artifacts to blob storage. publish: false # If true, publish artifacts to the public using Release Studio integration. @@ -25,21 +27,22 @@ parameters: stages: - ${{ if eq(parameters.publishExistingRunID, 'nil') }}: - - ${{ each builder in parameters.builders }}: - - template: pool.yml - parameters: - inner: - template: run-stage.yml - parameters: - builder: ${{ builder }} - createSourceArchive: ${{ parameters.createSourceArchive }} - releaseVersion: ${{ parameters.releaseVersion }} - official: ${{ parameters.official }} - createSymbols: ${{ parameters.createSymbols }} - # Attempt to retry the build on Windows to mitigate flakiness: - # "Access Denied" during EXE copying and general flakiness during tests. - ${{ if eq(builder.os, 'windows') }}: - retryAttempts: [1, 2, 3, 4, "FINAL"] + - ${{ if eq(parameters.signExistingRunID, 'nil') }}: + - ${{ each builder in parameters.builders }}: + - template: pool.yml + parameters: + inner: + template: run-stage.yml + parameters: + builder: ${{ builder }} + createSourceArchive: ${{ parameters.createSourceArchive }} + releaseVersion: ${{ parameters.releaseVersion }} + official: ${{ parameters.official }} + createSymbols: ${{ parameters.createSymbols }} + # Attempt to retry the build on Windows to mitigate flakiness: + # "Access Denied" during EXE copying and general flakiness during tests. + ${{ if eq(builder.os, 'windows') }}: + retryAttempts: [1, 2, 3, 4, "FINAL"] - ${{ if eq(parameters.sign, true) }}: - template: pool.yml @@ -55,6 +58,7 @@ stages: - ${{ each builder in parameters.builders }}: - ${{ if eq(builder.config, 'buildandpack') }}: - ${{ builder }} + signExistingRunID: ${{ parameters.signExistingRunID }} - ${{ if eq(parameters.publish, true) }}: - ${{ if and(not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')), eq(parameters.publishReleaseStudio, true)) }}: diff --git a/eng/pipeline/stages/go-builder-matrix-stages.yml b/eng/pipeline/stages/go-builder-matrix-stages.yml index c202248f02f..98253e5171a 100644 --- a/eng/pipeline/stages/go-builder-matrix-stages.yml +++ b/eng/pipeline/stages/go-builder-matrix-stages.yml @@ -31,6 +31,9 @@ parameters: - name: sign type: boolean default: false + - name: signExistingRunID + type: string + default: 'nil' - name: publish type: boolean default: false @@ -57,6 +60,7 @@ stages: jobsParameters: official: ${{ parameters.official }} sign: ${{ parameters.sign }} + signExistingRunID: ${{ parameters.signExistingRunID }} publish: ${{ parameters.publish }} publishReleaseStudio: ${{ parameters.publishReleaseStudio }} publishExistingRunID: ${{ parameters.publishExistingRunID }} @@ -71,6 +75,8 @@ stages: - { os: windows, arch: amd64, config: buildandpack } - { os: linux, arch: arm, hostArch: amd64, config: buildandpack } - { os: linux, arch: arm64, hostArch: amd64, config: buildandpack } + - { os: darwin, arch: amd64, config: buildandpack } + - { os: darwin, arch: arm64, hostArch: amd64, config: buildandpack } - ${{ if parameters.includeArm64Host }}: - { os: linux, arch: arm64, config: buildandpack } - ${{ if parameters.innerloop }}: diff --git a/eng/pipeline/stages/pool-1.yml b/eng/pipeline/stages/pool-1.yml index edb92053c6e..a53b4a72498 100644 --- a/eng/pipeline/stages/pool-1.yml +++ b/eng/pipeline/stages/pool-1.yml @@ -22,7 +22,9 @@ stages: parameters: ${{ insert }}: ${{ parameters }} - ${{ if eq(parameters.hostArch, 'arm64') }}: + ${{ if eq(parameters.os, 'darwin') }}: + name: Azure Pipelines # use the default AzDo hosted pool + ${{ elseif and(eq(parameters.hostArch, 'arm64'), eq(parameters.os, 'linux')) }}: name: Docker-Linux-Arm-Internal ${{ else }}: ${{ if parameters.public }}: diff --git a/eng/pipeline/stages/pool-2.yml b/eng/pipeline/stages/pool-2.yml index 5a59423e0ce..1509f1194d5 100644 --- a/eng/pipeline/stages/pool-2.yml +++ b/eng/pipeline/stages/pool-2.yml @@ -50,3 +50,8 @@ stages: ${{ else }}: demands: ImageOverride -equals 1es-ubuntu-2004 os: linux + + ${{ elseif eq(parameters.os, 'darwin') }}: + # https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml#software + vmImage: 'macos-14' + os: macOs diff --git a/eng/pipeline/stages/sign-stage.yml b/eng/pipeline/stages/sign-stage.yml index 73e1e046a58..2d50a05f009 100644 --- a/eng/pipeline/stages/sign-stage.yml +++ b/eng/pipeline/stages/sign-stage.yml @@ -6,15 +6,29 @@ # publishes the signed files and signatures into a consolidated pipeline artifact. parameters: + - name: builder + type: object + - name: official + type: boolean + - name: pool + type: object + # [] of { id, os, arch, config, distro?, experiment?, broken? } - builders: [] + - name: builders + type: object + + - name: signExistingRunID + type: string stages: - stage: Sign - # Depend on all build stages that produced artifacts that need signing. - dependsOn: - - ${{ each builder in parameters.builders }}: - - ${{ builder.id }} + ${{ if eq(parameters.signExistingRunID, 'nil') }}: + dependsOn: + # Depend on all build stages that produced artifacts that need signing. + - ${{ each builder in parameters.builders }}: + - ${{ builder.id }} + ${{ else }}: + dependsOn: [] jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - job: Sign @@ -22,6 +36,9 @@ stages: workspace: clean: all + # Give the sign task leeway to finish up after hitting its own timeout. + timeoutInMinutes: 80 + templateContext: mb: signing: @@ -32,46 +49,46 @@ stages: outputs: # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs - output: pipelineArtifact - path: $(Build.StagingDirectory)/ToSign + path: 'eng\signing\signed' artifact: Binaries Signed + + - output: pipelineArtifact + path: 'eng\signing\signing-temp' + artifact: Signing temp directory $(System.JobAttempt) + condition: always() + - output: pipelineArtifact path: 'eng\signing' - artifact: Signing diagnosis directory + artifact: Signing diagnosis directory $(System.JobAttempt) + condition: always() steps: - template: ../steps/checkout-windows-task.yml - ${{ each builder in parameters.builders }}: - - download: current - artifact: Binaries ${{ builder.id }} - # Filter out manifests added by 1ES pipeline template. - patterns: '!_manifest/**' - displayName: 'Download: Binaries ${{ builder.id }}' - - - powershell: | - $flatDir = "$(Build.StagingDirectory)/ToSign" - New-Item $flatDir -ItemType Directory -ErrorAction Ignore - - Get-ChildItem -Recurse -File -Path @( - 'Binaries ${{ builder.id }}' - ) | %{ - if (Test-Path "$flatDir\$($_.Name)") { - throw "Duplicate filename, unable to flatten: $($_.FullName)" - } - Copy-Item $_.FullName $flatDir - } - displayName: 'Copy to flat dir: ${{ builder.id }}' - workingDirectory: '$(Pipeline.Workspace)' + - ${{ if eq(parameters.signExistingRunID, 'nil') }}: + - download: current + artifact: Binaries ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + displayName: 'Download: Binaries ${{ builder.id }}' + - ${{ else }}: + - task: DownloadPipelineArtifact@2 + displayName: 'Download: Binaries ${{ builder.id }} (Specific)' + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + runVersion: 'specific' + runId: ${{ parameters.signExistingRunID }} + artifact: Binaries ${{ builder.id }} + # Filter out manifests added by 1ES pipeline template. + patterns: '!_manifest/**' + targetPath: '$(Pipeline.Workspace)/Binaries ${{ builder.id }}' - - task: DotNetCoreCLI@2 - displayName: 'Sign Files' - inputs: - command: custom - projects: '$(Build.SourcesDirectory)/eng/signing/Sign.proj' - custom: build - arguments: >- - /t:AfterBuild - /p:SignFilesDir=$(Build.StagingDirectory)/ToSign - /p:SignType=$(SignType) - /bl:eng/signing/SignFiles.binlog - /v:n + - pwsh: | + eng/run.ps1 sign ` + -files '$(Pipeline.Workspace)/Binaries */*' ` + -sign-type '$(SignType)' ` + -timeout 60m + displayName: Sign Files diff --git a/eng/signing/.gitignore b/eng/signing/.gitignore index c66ddb25baa..a784cf86ebb 100644 --- a/eng/signing/.gitignore +++ b/eng/signing/.gitignore @@ -9,3 +9,4 @@ obj/ signing-log/ signing-temp/ tosign/ +signed/ diff --git a/eng/signing/NuGet.config b/eng/signing/NuGet.config index 080b3fce698..8f32de83184 100644 --- a/eng/signing/NuGet.config +++ b/eng/signing/NuGet.config @@ -1,7 +1,7 @@ diff --git a/eng/signing/README.md b/eng/signing/README.md index c8d8879550d..2ffc00ab368 100644 --- a/eng/signing/README.md +++ b/eng/signing/README.md @@ -1,23 +1,8 @@ -# Signing infrastructure - -This directory contains the infrastructure used by Microsoft to sign the Go -binaries in internal builds. It uses -[MicroBuild Signing](https://dev.azure.com/devdiv/DevDiv/_wiki/wikis/DevDiv.wiki/650/MicroBuild-Signing) -(internal Microsoft wiki link). - -To see it in action, go to [`/eng/pipeline/README.md`](/eng/pipeline/README.md) -and follow the link for `microsoft-go`. - -This infrastructure runs on Windows only. - -## Running locally - -1. Create the directory `tosign` and add `.tar.gz` and `.zip` artifacts. -1. Install the plugin: - 1. Download the latest https://devdiv.visualstudio.com/DevDiv/_artifacts/feed/MicroBuildToolset/NuGet/MicroBuild.Plugins.Signing - 1. Extract it to `%userprofile%\.nuget\microbuild.plugins.signing\1.1.900`. - * Optionally make the last dir match the version of the package. It will be discovered dynamically, as a plugin, whether or not it matches. -1. Run a "test sign" build locally to exercise the tooling: - ``` - dotnet build /p:SignFilesDir=tosign /p:SignType=test /p:MicroBuild_SigningEnabled=true /bl - ``` +# MSBuild signing infrastructure + +This directory contains a component of the Microsoft Go signing infrastructure written using MSBuild. +`Sign.csproj` is the interface between the Go signing command [`/eng/_util/cmd/sign`][sign] and MicroBuild, an internal Microsoft toolset written to primarily support .NET projects that use MSBuild. + +See [`/eng/_util/cmd/sign`][sign] for more information about the signing infrastructure. + +[sign]: /eng/_util/cmd/sign \ No newline at end of file diff --git a/eng/signing/Sign.csproj b/eng/signing/Sign.csproj new file mode 100644 index 00000000000..2cca241e2bc --- /dev/null +++ b/eng/signing/Sign.csproj @@ -0,0 +1,28 @@ + + + + + + net7.0 + + + + + + + + + + + + + + $([MSBuild]::NormalizeDirectory('$(SignFilesDir)')) + + + + + + + + diff --git a/eng/signing/Sign.proj b/eng/signing/Sign.proj deleted file mode 100644 index 5f8ba9c8816..00000000000 --- a/eng/signing/Sign.proj +++ /dev/null @@ -1,93 +0,0 @@ - - - - - - net7.0 - - - - - - - - - - - - - true - false - - - - $([MSBuild]::NormalizeDirectory('$(SignFilesDir)')) - - - - - - - - LinuxSignManagedLanguageCompiler - - - - - - - - - - - - - - - - - - - - Microsoft400 - - - - - - - - - - - - - - - - - - - - -