Feat/default add look up tags REBASED

cmd/common.go

@@ -238,6 +238,24 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		return err
 	}
 
+	dumpConfig.LookUpSelectorTagsConsumers, err = determineLookUpSelectorTagsConsumers(*targetContent)
+	if err != nil {
+		return fmt.Errorf("error determining lookup selector tags: %w", err)
+	}
+
+	if dumpConfig.LookUpSelectorTagsConsumers != nil {
+		consumersGlobal, err := dump.GetAllConsumers(ctx, kongClient, dumpConfig.LookUpSelectorTagsConsumers)
+		if err != nil {
+			return fmt.Errorf("error retrieving global consumers via lookup selector tags: %w", err)
+		}
+		for _, c := range consumersGlobal {
+			targetContent.Consumers = append(targetContent.Consumers, file.FConsumer{Consumer: *c})
+			if err != nil {
+				return fmt.Errorf("error adding global consumer %v: %w", *c.Username, err)
+			}
+		}
+	}
+
 	if utils.Kong340Version.LTE(parsedKongVersion) {
 		dumpConfig.IsConsumerGroupScopedPluginSupported = true
 	}
@@ -321,6 +339,21 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	return nil
 }
 
+func determineLookUpSelectorTagsConsumers(targetContent file.Content) ([]string, error) {
+	if targetContent.Info != nil &&
+		targetContent.Info.LookUpSelectorTags != nil &&
+		targetContent.Info.LookUpSelectorTags.Consumers != nil {
+		if len(targetContent.Info.LookUpSelectorTags.Consumers) == 0 {
+			return nil, fmt.Errorf("global consumers specified but no global tags")
+		}
+		utils.RemoveDuplicates(&targetContent.Info.LookUpSelectorTags.Consumers)
+		sort.Strings(targetContent.Info.LookUpSelectorTags.Consumers)
+		return targetContent.Info.LookUpSelectorTags.Consumers, nil
+
+	}
+	return nil, nil
+}
+
 func determineSelectorTag(targetContent file.Content, config dump.Config) ([]string, error) {
 	if targetContent.Info != nil {
 		if len(targetContent.Info.SelectorTags) > 0 {

cmd/gateway_validate.go

@@ -47,6 +47,25 @@ func executeValidate(cmd *cobra.Command, _ []string) error {
 		if err != nil {
 			return err
 		}
+
+		// if this is an online validation, we need to look up upstream consumers if required.
+		lookUpSelectorTagsConsumers, err := determineLookUpSelectorTagsConsumers(*targetContent)
+		if err != nil {
+			return fmt.Errorf("error determining lookup selector tags: %w", err)
+		}
+
+		if lookUpSelectorTagsConsumers != nil {
+			consumersGlobal, err := dump.GetAllConsumers(ctx, kongClient, lookUpSelectorTagsConsumers)
+			if err != nil {
+				return fmt.Errorf("error retrieving global consumers via lookup selector tags: %w", err)
+			}
+			for _, c := range consumersGlobal {
+				targetContent.Consumers = append(targetContent.Consumers, file.FConsumer{Consumer: *c})
+				if err != nil {
+					return fmt.Errorf("error adding global consumer %v: %w", *c.Username, err)
+				}
+			}
+		}
 	}
 
 	rawState, err := file.Get(ctx, targetContent, file.RenderConfig{

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/fatih/color v1.15.0
 	github.com/google/go-cmp v0.6.0
 	github.com/kong/go-apiops v0.1.27
-	github.com/kong/go-database-reconciler v1.1.0
+	github.com/kong/go-database-reconciler v1.2.0
 	github.com/kong/go-kong v0.50.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.8.0
@@ -92,7 +92,7 @@ require (
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
-	github.com/shirou/gopsutil/v3 v3.23.10 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect

go.sum

@@ -182,8 +182,8 @@ github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y7
 github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/kong/go-apiops v0.1.27 h1:Jy9HSBtGtO0r0UpXSFenCfOUA0QmeX4i8iGQU0vnC+Q=
 github.com/kong/go-apiops v0.1.27/go.mod h1:TYRNVbQ/lw6D3AUJBVP1w4zmMlJ59K83q+zCFa02uZ4=
-github.com/kong/go-database-reconciler v1.1.0 h1:USCdsAj/7eh9sOOfbnvsOe4jw5k4+FSTD3okcTLIVqQ=
-github.com/kong/go-database-reconciler v1.1.0/go.mod h1:p8NvafqBSuMR9YNCOZ24aIeeajc145+biXpAaMExvpI=
+github.com/kong/go-database-reconciler v1.2.0 h1:deqgEy4txd8VayuFGdW3nyiOwO3NvVLyTDIIgFeK6fo=
+github.com/kong/go-database-reconciler v1.2.0/go.mod h1:dyklGuvQFgJD3nGLndQbuNoE3E3184QKjVmhNpDDdQw=
 github.com/kong/go-kong v0.50.0 h1:HDKn3o/02AH4cURvjzS09gqC4bHZDva5H8JJwYi7T1U=
 github.com/kong/go-kong v0.50.0/go.mod h1:xDf1RfkaE/rAwNE1fS3XniFj/d2JmkEER2S9NDY12Yw=
 github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY=
@@ -285,8 +285,8 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPO
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
 github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM=
-github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE=
+github.com/shirou/gopsutil/v3 v3.23.12 h1:z90NtUkp3bMtmICZKpC4+WaknU1eXtp5vtbQ11DgpE4=
+github.com/shirou/gopsutil/v3 v3.23.12/go.mod h1:1FrWgea594Jp7qmjHUUPlJDTPgcsb9mGnXDxavtikzM=
 github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
 github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
 github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
@@ -418,7 +418,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

tests/integration/sync_test.go

@@ -4715,6 +4715,44 @@ func Test_Sync_DoNotUpdateCreatedAt(t *testing.T) {
 	// consumers do not have an updated_at field
 }
 
+// Test_Sync_LookupConsumerTags tests that existing behavior when referencing
+// consumers from plugins is preserved:
+// - if a referenced consumer is not present in the state file, the sync fails
+// - if a referenced consumer is present in the state file, the sync succeeds
+//
+// This test also tests that the new behavior is implemented correctly:
+//   - if a referenced consumer is not present in the state file, but is present
+//     in Kong when using the new lookup selector tags, the sync succeeds
+//   - if a referenced consumer is not present in the state file and neither in
+//     Kong when using the new lookup selector tags, the sync fails
+func Test_Sync_LookupConsumerTags(t *testing.T) {
+	runWhen(t, "enterprise", ">=3.0.0")
+	setup(t)
+
+	// test that reference to non-existing consumer fails.
+	pluginsNoLookupStateFile := "testdata/sync/029-lookup-tags/plugins_no_lookup.yaml"
+	err := sync(pluginsNoLookupStateFile)
+	require.Error(t, err)
+	require.EqualError(t, err, "building state: consumer foo for plugin rate-limiting-advanced: entity not found")
+
+	// test that reference to existing local consumer succeeds.
+	pluginsAndConsumersStateFile := "testdata/sync/029-lookup-tags/plugins_and_consumers.yaml"
+	require.NoError(t, sync(pluginsAndConsumersStateFile))
+	reset(t)
+
+	// test that reference to existing global consumer succeeds via lookup tags.
+	globalConsumersStateFile := "testdata/sync/029-lookup-tags/global_consumers.yaml"
+	require.NoError(t, sync(globalConsumersStateFile))
+	// sync plugins with lookup reference to global consumers.
+	pluginsLookupStateFile := "testdata/sync/029-lookup-tags/plugins_lookup.yaml"
+	require.NoError(t, sync(pluginsLookupStateFile))
+	reset(t)
+
+	// test that reference to non-existing global consumer fails via lookup tags.
+	require.Error(t, sync(pluginsLookupStateFile))
+	require.EqualError(t, err, "building state: consumer foo for plugin rate-limiting-advanced: entity not found")
+}
+
 // test scope:
 //   - 3.0.0+
 //   - konnect

tests/integration/testdata/sync/029-lookup-tags/global_consumers.yaml

@@ -0,0 +1,6 @@
+_format_version: "3.0"
+_info:
+  select_tags:
+  - global-consumers
+consumers:
+- username: foo

tests/integration/testdata/sync/029-lookup-tags/plugins_and_consumers.yaml

@@ -0,0 +1,13 @@
+_format_version: "3.0"
+plugins:
+- name: rate-limiting-advanced
+  config:
+    limit:
+    - 10
+    window_size:
+    - 60
+    namespace: foo
+    sync_rate: -1
+  consumer: foo
+consumers:
+- username: foo

tests/integration/testdata/sync/029-lookup-tags/plugins_lookup.yaml

@@ -0,0 +1,17 @@
+_format_version: "3.0"
+_info:
+  select_tags:
+  - managed-by-deck
+  default_lookup_tags:
+    consumers:
+    - global-consumers
+plugins:
+- name: rate-limiting-advanced
+  config:
+    limit:
+    - 10
+    window_size:
+    - 60
+    namespace: foo
+    sync_rate: -1
+  consumer: foo

tests/integration/testdata/sync/029-lookup-tags/plugins_no_lookup.yaml

@@ -0,0 +1,14 @@
+_format_version: "3.0"
+_info:
+  select_tags:
+  - managed-by-deck
+plugins:
+- name: rate-limiting-advanced
+  config:
+    limit:
+    - 10
+    window_size:
+    - 60
+    namespace: foo
+    sync_rate: -1
+  consumer: foo