From a5d7465beb45885350f0cbc3de4a7d9849a13cdf Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 16:30:30 -0500 Subject: [PATCH 1/6] Add `buf plugin prune` command --- private/buf/cmd/buf/buf.go | 2 + .../command/plugin/pluginprune/pluginprune.go | 117 ++++++++++++++++++ .../command/plugin/pluginprune/usage.gen.go | 19 +++ 3 files changed, 138 insertions(+) create mode 100644 private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go create mode 100644 private/buf/cmd/buf/command/plugin/pluginprune/usage.gen.go diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index cc8e61a481..5febab7e33 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -62,6 +62,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlsbreakingrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modlslintrules" "github.com/bufbuild/buf/private/buf/cmd/buf/command/mod/modopen" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginprune" "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginpush" "github.com/bufbuild/buf/private/buf/cmd/buf/command/plugin/pluginupdate" "github.com/bufbuild/buf/private/buf/cmd/buf/command/push" @@ -191,6 +192,7 @@ func NewRootCommand(name string) *appcmd.Command { SubCommands: []*appcmd.Command{ pluginpush.NewCommand("push", builder), pluginupdate.NewCommand("update", builder), + pluginprune.NewCommand("prune", builder), }, }, { diff --git a/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go new file mode 100644 index 0000000000..d649d26e53 --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go @@ -0,0 +1,117 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pluginprune + +import ( + "context" + + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/bufworkspace" + "github.com/bufbuild/buf/private/bufpkg/bufplugin" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/slicesext" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appext.SubCommandBuilder, +) *appcmd.Command { + return &appcmd.Command{ + Use: name + " ", + Short: "Prune unused plugins from buf.lock", + Long: `Plugins that are no longer configured in buf.yaml are removed from the buf.lock file. + +The first argument is the directory of your buf.yaml configuration file. +Defaults to "." if no argument is specified.`, + Args: appcmd.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appext.Container) error { + return run(ctx, container) + }, + ), + } +} + +func run( + ctx context.Context, + container appext.Container, +) error { + dirPath := "." + if container.NumArgs() > 0 { + dirPath = container.Arg(0) + } + controller, err := bufcli.NewController(container) + if err != nil { + return err + } + workspaceDepManager, err := controller.GetWorkspaceDepManager(ctx, dirPath) + if err != nil { + return err + } + configuredRemotePluginRefs, err := workspaceDepManager.ConfiguredRemotePluginRefs(ctx) + if err != nil { + return err + } + pluginKeyProvider, err := bufcli.NewPluginKeyProvider(container) + if err != nil { + return err + } + configuredRemotePluginKeys, err := pluginKeyProvider.GetPluginKeysForPluginRefs( + ctx, + configuredRemotePluginRefs, + bufplugin.DigestTypeP1, + ) + if err != nil { + return err + } + return prune( + ctx, + slicesext.Map( + configuredRemotePluginKeys, + func(pluginKey bufplugin.PluginKey) string { + return pluginKey.String() + }, + ), + workspaceDepManager, + ) +} + +func prune( + ctx context.Context, + bufYAMLBasedRemotePluginKeys []string, + workspaceDepManager bufworkspace.WorkspaceDepManager, +) error { + bufYAMLPluginKeys := slicesext.ToStructMap(bufYAMLBasedRemotePluginKeys) + existingRemotePluginKeys, err := workspaceDepManager.ExistingBufLockFileRemotePluginKeys(ctx) + if err != nil { + return err + } + var updatedBufLockPluginKeys []bufplugin.PluginKey + for _, existingRemotePluginKey := range existingRemotePluginKeys { + // Check if an existing plugin key from the buf.lock is confiugred in the buf.yaml. + if _, ok := bufYAMLPluginKeys[existingRemotePluginKey.String()]; ok { + // If yes, then we keep it for the updated buf.lock. + updatedBufLockPluginKeys = append(updatedBufLockPluginKeys, existingRemotePluginKey) + } + } + // We keep the existing dep module keys as-is. + existingDepModuleKeys, err := workspaceDepManager.ExistingBufLockFileDepModuleKeys(ctx) + if err != nil { + return err + } + return workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, updatedBufLockPluginKeys) +} diff --git a/private/buf/cmd/buf/command/plugin/pluginprune/usage.gen.go b/private/buf/cmd/buf/command/plugin/pluginprune/usage.gen.go new file mode 100644 index 0000000000..9f84b1ce9e --- /dev/null +++ b/private/buf/cmd/buf/command/plugin/pluginprune/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2024 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package pluginprune + +import _ "github.com/bufbuild/buf/private/usage" From 96d456f962fe43142ca8f4cac7c52e05090bf1a2 Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 17:33:21 -0500 Subject: [PATCH 2/6] Fix update so it doesn't prune --- .../plugin/pluginupdate/pluginupdate.go | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go index 3965bb4cee..0e645716dc 100644 --- a/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go +++ b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go @@ -18,11 +18,13 @@ import ( "context" "errors" "fmt" + "sort" "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" + "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/spf13/pflag" ) @@ -131,6 +133,31 @@ func run( if err != nil { return err } + // We do not prune the buf.lock during update, we only add update existing plugin keys + // or add new plugin keys. + configuredRemotePluginKeyNameToPluginKey := map[string]bufplugin.PluginKey{} + for _, configuredRemotePluginKey := range configuredRemotePluginKeys { + configuredRemotePluginKeyNameToPluginKey[configuredRemotePluginKey.FullName().String()] = configuredRemotePluginKey + } + var updatedRemotePluginKeys []bufplugin.PluginKey + for _, existingRemotePluginKey := range existingRemotePluginKeys { + if updatedRemotePluginKey, ok := configuredRemotePluginKeyNameToPluginKey[existingRemotePluginKey.FullName().String()]; ok { + updatedRemotePluginKeys = append(updatedRemotePluginKeys, updatedRemotePluginKey) + // This was found and updated, so we can delete from the map + delete(configuredRemotePluginKeyNameToPluginKey, existingRemotePluginKey.FullName().String()) + } else { + // Otherwise, keep the existing remote plugin key from buf.lock + updatedRemotePluginKeys = append(updatedRemotePluginKeys, existingRemotePluginKey) + } + } + // Add any new configured remote plugin keys + updatedRemotePluginKeys = append(updatedRemotePluginKeys, slicesext.MapValuesToSlice(configuredRemotePluginKeyNameToPluginKey)...) + sort.Slice( + updatedRemotePluginKeys, + func(i int, j int) bool { + return updatedRemotePluginKeys[i].FullName().String() < updatedRemotePluginKeys[j].FullName().String() + }, + ) // We're about to edit the buf.lock file on disk. If we have a subsequent error, // attempt to revert the buf.lock file. @@ -144,7 +171,7 @@ func run( } }() // Edit the buf.lock file with the updated remote plugins. - if err := workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, configuredRemotePluginKeys); err != nil { + if err := workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, updatedRemotePluginKeys); err != nil { return err } return nil From 1c8e6fa3df795c7e266b8593457344fb7a18d49f Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 17:43:07 -0500 Subject: [PATCH 3/6] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e1153d490..545e830e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ determinisitic. - Add `buf plugin push` command to push a plugin to the Buf Schema Registry. Only WebAssembly check plugins are supported at this time. +- Add `buf plugin update` and `buf plugin prune` command to manage plugins in the `buf.lock` + file. Only WebAssembly check plugins are supported at this time. - Add `buf registry plugin commit {add-label,info,list,resolve}` to manage BSR plugin commits. - Add `buf registry plugin label {archive,info,list,unarchive}` to manage BSR plugin commits. From 7b489378b9a8e68165f03741ea22dd4916c48c2d Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 18:00:04 -0500 Subject: [PATCH 4/6] Revert changes to `buf plugin update` --- .../plugin/pluginupdate/pluginupdate.go | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go index 0e645716dc..3965bb4cee 100644 --- a/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go +++ b/private/buf/cmd/buf/command/plugin/pluginupdate/pluginupdate.go @@ -18,13 +18,11 @@ import ( "context" "errors" "fmt" - "sort" "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" - "github.com/bufbuild/buf/private/pkg/slicesext" "github.com/bufbuild/buf/private/pkg/syserror" "github.com/spf13/pflag" ) @@ -133,31 +131,6 @@ func run( if err != nil { return err } - // We do not prune the buf.lock during update, we only add update existing plugin keys - // or add new plugin keys. - configuredRemotePluginKeyNameToPluginKey := map[string]bufplugin.PluginKey{} - for _, configuredRemotePluginKey := range configuredRemotePluginKeys { - configuredRemotePluginKeyNameToPluginKey[configuredRemotePluginKey.FullName().String()] = configuredRemotePluginKey - } - var updatedRemotePluginKeys []bufplugin.PluginKey - for _, existingRemotePluginKey := range existingRemotePluginKeys { - if updatedRemotePluginKey, ok := configuredRemotePluginKeyNameToPluginKey[existingRemotePluginKey.FullName().String()]; ok { - updatedRemotePluginKeys = append(updatedRemotePluginKeys, updatedRemotePluginKey) - // This was found and updated, so we can delete from the map - delete(configuredRemotePluginKeyNameToPluginKey, existingRemotePluginKey.FullName().String()) - } else { - // Otherwise, keep the existing remote plugin key from buf.lock - updatedRemotePluginKeys = append(updatedRemotePluginKeys, existingRemotePluginKey) - } - } - // Add any new configured remote plugin keys - updatedRemotePluginKeys = append(updatedRemotePluginKeys, slicesext.MapValuesToSlice(configuredRemotePluginKeyNameToPluginKey)...) - sort.Slice( - updatedRemotePluginKeys, - func(i int, j int) bool { - return updatedRemotePluginKeys[i].FullName().String() < updatedRemotePluginKeys[j].FullName().String() - }, - ) // We're about to edit the buf.lock file on disk. If we have a subsequent error, // attempt to revert the buf.lock file. @@ -171,7 +144,7 @@ func run( } }() // Edit the buf.lock file with the updated remote plugins. - if err := workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, updatedRemotePluginKeys); err != nil { + if err := workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, configuredRemotePluginKeys); err != nil { return err } return nil From 7a9406dc0d8b060b4c6752545bdef9aec6579430 Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 18:03:56 -0500 Subject: [PATCH 5/6] Rename var --- .../buf/cmd/buf/command/plugin/pluginprune/pluginprune.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go index d649d26e53..304dadfd66 100644 --- a/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go +++ b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go @@ -100,12 +100,12 @@ func prune( if err != nil { return err } - var updatedBufLockPluginKeys []bufplugin.PluginKey + var prunedBufLockPluginKeys []bufplugin.PluginKey for _, existingRemotePluginKey := range existingRemotePluginKeys { // Check if an existing plugin key from the buf.lock is confiugred in the buf.yaml. if _, ok := bufYAMLPluginKeys[existingRemotePluginKey.String()]; ok { // If yes, then we keep it for the updated buf.lock. - updatedBufLockPluginKeys = append(updatedBufLockPluginKeys, existingRemotePluginKey) + prunedBufLockPluginKeys = append(prunedBufLockPluginKeys, existingRemotePluginKey) } } // We keep the existing dep module keys as-is. @@ -113,5 +113,5 @@ func prune( if err != nil { return err } - return workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, updatedBufLockPluginKeys) + return workspaceDepManager.UpdateBufLockFile(ctx, existingDepModuleKeys, prunedBufLockPluginKeys) } From 38968ae06aaea6f95d2fc153d9f826a3146fb9f7 Mon Sep 17 00:00:00 2001 From: Doria Keung Date: Mon, 9 Dec 2024 18:11:01 -0500 Subject: [PATCH 6/6] No need to resolve keys, use refs directly --- .../command/plugin/pluginprune/pluginprune.go | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go index 304dadfd66..f2dd2769c8 100644 --- a/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go +++ b/private/buf/cmd/buf/command/plugin/pluginprune/pluginprune.go @@ -19,6 +19,7 @@ import ( "github.com/bufbuild/buf/private/buf/bufcli" "github.com/bufbuild/buf/private/buf/bufworkspace" + "github.com/bufbuild/buf/private/bufpkg/bufparse" "github.com/bufbuild/buf/private/bufpkg/bufplugin" "github.com/bufbuild/buf/private/pkg/app/appcmd" "github.com/bufbuild/buf/private/pkg/app/appext" @@ -66,24 +67,12 @@ func run( if err != nil { return err } - pluginKeyProvider, err := bufcli.NewPluginKeyProvider(container) - if err != nil { - return err - } - configuredRemotePluginKeys, err := pluginKeyProvider.GetPluginKeysForPluginRefs( - ctx, - configuredRemotePluginRefs, - bufplugin.DigestTypeP1, - ) - if err != nil { - return err - } return prune( ctx, slicesext.Map( - configuredRemotePluginKeys, - func(pluginKey bufplugin.PluginKey) string { - return pluginKey.String() + configuredRemotePluginRefs, + func(pluginRef bufparse.Ref) string { + return pluginRef.FullName().String() }, ), workspaceDepManager, @@ -92,10 +81,10 @@ func run( func prune( ctx context.Context, - bufYAMLBasedRemotePluginKeys []string, + bufYAMLBasedRemotePluginNames []string, workspaceDepManager bufworkspace.WorkspaceDepManager, ) error { - bufYAMLPluginKeys := slicesext.ToStructMap(bufYAMLBasedRemotePluginKeys) + bufYAMLRemotePluginNames := slicesext.ToStructMap(bufYAMLBasedRemotePluginNames) existingRemotePluginKeys, err := workspaceDepManager.ExistingBufLockFileRemotePluginKeys(ctx) if err != nil { return err @@ -103,7 +92,7 @@ func prune( var prunedBufLockPluginKeys []bufplugin.PluginKey for _, existingRemotePluginKey := range existingRemotePluginKeys { // Check if an existing plugin key from the buf.lock is confiugred in the buf.yaml. - if _, ok := bufYAMLPluginKeys[existingRemotePluginKey.String()]; ok { + if _, ok := bufYAMLRemotePluginNames[existingRemotePluginKey.FullName().String()]; ok { // If yes, then we keep it for the updated buf.lock. prunedBufLockPluginKeys = append(prunedBufLockPluginKeys, existingRemotePluginKey) }