Add a utility to determine latest plugins (#945)

Add a new `latest-plugins` CLI to determine the latest published plugins
(and their dependencies) and output them as a JSON document to stdout.
This will be used downstream by other utilities to package plugins for
distribution.

cmd/latest-plugins/main.go

@@ -0,0 +1,129 @@
+/*
+latest-plugins outputs the latest non-community plugins (and their dependencies) in JSON format to stdout.
+To determine available plugins, it downloads the plugin-releases.json file from the latest bufbuild/plugins release.
+Additionally, it verifies the contents of the file against the minisign signature.
+
+This utility is used downstream by some other tooling to package up plugins to install in the BSR.
+*/
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"aead.dev/minisign"
+	"github.com/bufbuild/buf/private/pkg/interrupt"
+	"golang.org/x/mod/semver"
+
+	"github.com/bufbuild/plugins/internal/release"
+)
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatalf("failed to run: %v", err)
+	}
+}
+
+func run() error {
+	ctx, cancel := interrupt.WithCancel(context.Background())
+	defer cancel()
+	client := release.NewClient(ctx)
+	latestRelease, err := client.GetLatestRelease(ctx, release.GithubOwnerBufbuild, release.GithubRepoPlugins)
+	if err != nil {
+		return fmt.Errorf("failed to determine latest %s/%s release: %w", release.GithubOwnerBufbuild, release.GithubRepoPlugins, err)
+	}
+	releasesBytes, _, err := client.DownloadAsset(ctx, latestRelease, release.PluginReleasesFile)
+	if err != nil {
+		return fmt.Errorf("failed to download %s: %w", release.PluginReleasesFile, err)
+	}
+	releasesMinisigBytes, _, err := client.DownloadAsset(ctx, latestRelease, release.PluginReleasesSignatureFile)
+	if err != nil {
+		return fmt.Errorf("failed to download %s: %w", release.PluginReleasesSignatureFile, err)
+	}
+	publicKey, err := release.DefaultPublicKey()
+	if err != nil {
+		return fmt.Errorf("failed to load minisign public key: %w", err)
+	}
+	if !minisign.Verify(publicKey, releasesBytes, releasesMinisigBytes) {
+		return errors.New("failed to verify plugin-releases.json")
+	}
+	var pluginReleases release.PluginReleases
+	if err := json.NewDecoder(bytes.NewReader(releasesBytes)).Decode(&pluginReleases); err != nil {
+		return err
+	}
+	latestPlugins, err := getLatestPluginsAndDependencies(&pluginReleases)
+	if err != nil {
+		return fmt.Errorf("failed to determine latest plugins and dependencies: %w", err)
+	}
+	// sort by dependency order
+	sortedPlugins, err := release.SortReleasesInDependencyOrder(latestPlugins)
+	if err != nil {
+		return fmt.Errorf("failed to sort plugins in dependency order: %w", err)
+	}
+	return json.NewEncoder(os.Stdout).Encode(&release.PluginReleases{Releases: sortedPlugins})
+}
+
+func getLatestPluginsAndDependencies(releases *release.PluginReleases) ([]release.PluginRelease, error) {
+	versionToPlugin := make(map[string]release.PluginRelease, len(releases.Releases))
+	latestVersions := make(map[string]release.PluginRelease)
+	for _, pluginRelease := range releases.Releases {
+		owner, pluginName, found := strings.Cut(pluginRelease.PluginName, "/")
+		if !found {
+			return nil, fmt.Errorf("failed to split plugin pluginName into owner/pluginName")
+		}
+		switch owner {
+		case "community": // Disable community plugins by default
+			continue
+		case "bufbuild": // Don't include deprecated plugins.
+			switch pluginName {
+			case "connect-es",
+				"connect-go",
+				"connect-kotlin",
+				"connect-query",
+				"connect-swift",
+				"connect-swift-mocks",
+				"connect-web":
+				continue
+			}
+		}
+		versionToPlugin[pluginRelease.PluginName+":"+pluginRelease.PluginVersion] = pluginRelease
+		latestVersion, ok := latestVersions[pluginRelease.PluginName]
+		if !ok || semver.Compare(latestVersion.PluginVersion, pluginRelease.PluginVersion) < 0 {
+			latestVersions[pluginRelease.PluginName] = pluginRelease
+		}
+	}
+	toInclude := make(map[string]struct{})
+	deps := make(map[string]struct{})
+	for _, pluginRelease := range latestVersions {
+		toInclude[pluginRelease.PluginName+":"+pluginRelease.PluginVersion] = struct{}{}
+		for _, d := range pluginRelease.Dependencies {
+			deps[strings.TrimPrefix(d, "buf.build/")] = struct{}{}
+		}
+	}
+	for len(deps) > 0 {
+		nextDeps := make(map[string]struct{})
+		for dep := range deps {
+			if _, ok := toInclude[dep]; ok {
+				continue
+			}
+			toInclude[dep] = struct{}{}
+			for _, nextDep := range versionToPlugin[dep].Dependencies {
+				nextDeps[strings.TrimPrefix(nextDep, "buf.build/")] = struct{}{}
+			}
+		}
+		deps = nextDeps
+	}
+	var latestPluginsAndDeps []release.PluginRelease
+	for _, pluginRelease := range releases.Releases {
+		if _, ok := toInclude[pluginRelease.PluginName+":"+pluginRelease.PluginVersion]; ok {
+			latestPluginsAndDeps = append(latestPluginsAndDeps, pluginRelease)
+		}
+	}
+	return latestPluginsAndDeps, nil
+}

internal/release/release.go

@@ -3,9 +3,11 @@ package release
 import (
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
 	"io"
 	"log"
 	"os"
+	"strings"
 	"time"
 )
 
@@ -61,3 +63,38 @@ func CalculateDigest(path string) (string, error) {
 	hashBytes := hash.Sum(nil)
 	return "sha256:" + hex.EncodeToString(hashBytes), nil
 }
+
+// SortReleasesInDependencyOrder sorts the list of plugin releases so that a plugin's dependencies come before each plugin.
+// The original slice is unmodified - it returns a copy in sorted order, or an error if there is a cycle or unmet dependency.
+func SortReleasesInDependencyOrder(original []PluginRelease) ([]PluginRelease, error) {
+	// Make a defensive copy of the original list
+	plugins := make([]PluginRelease, len(original))
+	copy(plugins, original)
+	resolved := make([]PluginRelease, 0, len(plugins))
+	resolvedMap := make(map[string]struct{}, len(plugins))
+	for len(plugins) > 0 {
+		var unresolved []PluginRelease
+		for _, plugin := range plugins {
+			foundDeps := true
+			for _, dep := range plugin.Dependencies {
+				// TODO: This is kinda ugly - we don't include the remote on names in plugin-releases.json but do on deps.
+				if _, ok := resolvedMap[strings.TrimPrefix(dep, "buf.build/")]; !ok {
+					foundDeps = false
+					break
+				}
+			}
+			if foundDeps {
+				resolved = append(resolved, plugin)
+				resolvedMap[plugin.PluginName+":"+plugin.PluginVersion] = struct{}{}
+			} else {
+				unresolved = append(unresolved, plugin)
+			}
+		}
+		// We either have a cycle or a bug in dependency calculation
+		if len(unresolved) == len(plugins) {
+			return nil, fmt.Errorf("failed to resolve dependencies: %v", unresolved)
+		}
+		plugins = unresolved
+	}
+	return resolved, nil
+}