diff --git a/pkg/config/config.go b/pkg/config/config.go index 4fc9554..7e78a2d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -39,6 +39,7 @@ func Init() { NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.") NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.") NewBool(&ApplicationConfig.OverwriteConfigurationOnRestart, "overwrite-configuration-on-restart", false, "Overwrite the configuration in port on restarting the exporter. Optional.") + NewBool(&ApplicationConfig.UpdateEntityOnlyOnDiff, "update-entity-only-on-diff", false, "Optimization to reduce requests to port. Optional.") // Deprecated NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.") diff --git a/pkg/config/models.go b/pkg/config/models.go index 8020d39..fe3e99d 100644 --- a/pkg/config/models.go +++ b/pkg/config/models.go @@ -26,4 +26,5 @@ type ApplicationConfiguration struct { Resources []port.Resource DeleteDependents bool `json:"deleteDependents,omitempty"` CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"` + UpdateEntityOnlyOnDiff bool `json:"updateEntityOnlyOnDiff,omitempty"` } diff --git a/pkg/k8s/controller.go b/pkg/k8s/controller.go index 4da895b..1284021 100644 --- a/pkg/k8s/controller.go +++ b/pkg/k8s/controller.go @@ -10,11 +10,14 @@ import ( "github.com/port-labs/port-k8s-exporter/pkg/port" "github.com/port-labs/port-k8s-exporter/pkg/port/cli" "github.com/port-labs/port-k8s-exporter/pkg/port/mapping" + "hash/fnv" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/informers" + "strconv" + "encoding/json" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" @@ -69,7 +72,10 @@ func NewController(resource port.AggregatedResource, portClient *cli.PortClient, item.ActionType = UpdateAction item.Key, err = cache.MetaNamespaceKeyFunc(new) if err == nil { - controller.workqueue.Add(item) + + if controller.shouldSendUpdateEvent(old, new, config.ApplicationConfig.UpdateEntityOnlyOnDiff) { + controller.workqueue.Add(item) + } } }, DeleteFunc: func(obj interface{}) { @@ -246,7 +252,7 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, entities := make([]port.Entity, 0, len(mappings)) objectsToMap := make([]interface{}, 0) - if (itemsToParse == "") { + if itemsToParse == "" { objectsToMap = append(objectsToMap, structuredObj) } else { items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj) @@ -268,7 +274,7 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, objectsToMap = append(objectsToMap, copiedObject) } } - + for _, objectToMap := range objectsToMap { selectorResult, err := isPassSelector(objectToMap, selector) @@ -384,3 +390,53 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) { return k8sEntitiesSet, nil } + +func hashAllEntities(entities []port.Entity) (string, error) { + h := fnv.New64a() + for _, entity := range entities { + entityBytes, err := json.Marshal(entity) + if err != nil { + return "", err + } + _, err = h.Write(entityBytes) + if err != nil { + return "", err + } + } + return strconv.FormatUint(h.Sum64(), 10), nil +} + +func (c *Controller) shouldSendUpdateEvent(old interface{}, new interface{}, updateEntityOnlyOnDiff bool) bool { + + if updateEntityOnlyOnDiff == false { + return true + } + for _, kindConfig := range c.resource.KindConfigs { + oldEntities, err := c.getObjectEntities(old, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse) + if err != nil { + klog.Errorf("Error getting old entities: %v", err) + return true + } + newEntities, err := c.getObjectEntities(new, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse) + if err != nil { + klog.Errorf("Error getting new entities: %v", err) + return true + } + oldEntitiesHash, err := hashAllEntities(oldEntities) + if err != nil { + klog.Errorf("Error hashing old entities: %v", err) + return true + } + newEntitiesHash, err := hashAllEntities(newEntities) + if err != nil { + klog.Errorf("Error hashing new entities: %v", err) + return true + } + + if oldEntitiesHash != newEntitiesHash { + return true + } + } + + return false +} diff --git a/pkg/k8s/controller_test.go b/pkg/k8s/controller_test.go index 5f1737b..18e2f09 100644 --- a/pkg/k8s/controller_test.go +++ b/pkg/k8s/controller_test.go @@ -3,6 +3,7 @@ package k8s import ( "context" "fmt" + "github.com/stretchr/testify/assert" "reflect" "strings" "testing" @@ -19,6 +20,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/tools/cache" @@ -102,6 +104,40 @@ func newDeployment() *appsv1.Deployment { } } +func newDeploymentWithCustomLabels(generation int64, + generateName string, + creationTimestamp v1.Time, + labels map[string]string, +) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "port-k8s-exporter", + Namespace: "port-k8s-exporter", + GenerateName: generateName, + Generation: generation, + CreationTimestamp: creationTimestamp, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "port-k8s-exporter", + Image: "port-k8s-exporter:latest", + }, + }, + }, + }, + }, + } +} + func newUnstructured(obj interface{}) *unstructured.Unstructured { res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) if err != nil { @@ -317,3 +353,97 @@ func TestGetEntitiesSet(t *testing.T) { f := newFixture(t, "", "", "", resource, objects) f.runControllerGetEntitiesSet(expectedEntitiesSet, false) } + +func TestUpdateHandlerWithIndividualPropertyChanges(t *testing.T) { + + type Property struct { + Value interface{} + ShouldSendEvent bool + } + + fullMapping := []port.Resource{ + newResource("", []port.EntityMapping{ + { + Identifier: ".metadata.name", + Blueprint: "\"k8s-export-test-bp\"", + Icon: "\"Microservice\"", + Team: "\"Test\"", + Properties: map[string]string{ + "labels": ".spec.selector", + "generation": ".metadata.generation", + "generateName": ".metadata.generateName", + "creationTimestamp": ".metadata.creationTimestamp", + }, + Relations: map[string]string{ + "k8s-relation": "\"e_AgPMYvq1tAs8TuqM\"", + }, + }, + }), + newResource("", []port.EntityMapping{ + + { + Identifier: ".metadata.name", + Blueprint: "\"k8s-export-test-bp\"", + Icon: "\"Microservice\"", + Team: "\"Test\"", + Properties: map[string]string{}, + Relations: map[string]string{}, + }, + { + Identifier: ".metadata.name", + Blueprint: "\"k8s-export-test-bp\"", + Icon: "\"Microservice\"", + Team: "\"Test\"", + Properties: map[string]string{ + "labels": ".spec.selector", + "generation": ".metadata.generation", + "generateName": ".metadata.generateName", + "creationTimestamp": ".metadata.creationTimestamp", + }, + Relations: map[string]string{ + "k8s-relation": "\"e_AgPMYvq1tAs8TuqM\"", + }, + }, + }), + } + + for _, mapping := range fullMapping { + + controllerWithFullMapping := newFixture(t, "", "", "", mapping, []runtime.Object{}).controller + + // Test changes in each individual property + properties := map[string]Property{ + "metadata.name": {Value: "port-k8s-exporter", ShouldSendEvent: false}, + "something_without_mapping": {Value: "port-k8s-exporter", ShouldSendEvent: false}, + "metadata.generation": {Value: int64(3), ShouldSendEvent: true}, + "metadata.generateName": {Value: "new-port-k8s-exporter2", ShouldSendEvent: true}, + "metadata.creationTimestamp": {Value: v1.Now().Add(1 * time.Hour).Format(time.RFC3339), ShouldSendEvent: true}, + } + + for property, value := range properties { + newDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"})) + oldDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"})) + + // Update the property in the new deployment + unstructured.SetNestedField(newDep.Object, value.Value, strings.Split(property, ".")...) + + result := controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, true) + if value.ShouldSendEvent { + assert.True(t, result, fmt.Sprintf("Expected true when %s changes and feature flag is on", property)) + } else { + assert.False(t, result, fmt.Sprintf("Expected false when %s changes and feature flag is on", property)) + } + result = controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, false) + assert.True(t, result, fmt.Sprintf("Expected true when %s changes and feature flag is off", property)) + } + + // Add a case for json update because you can't edit the json directly + newDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"})) + oldDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "new-port-k8s-exporter"})) + + result := controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, true) + assert.True(t, result, fmt.Sprintf("Expected true when labels changes and feature flag is on")) + result = controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, false) + assert.True(t, result, fmt.Sprintf("Expected true when labels changes and feature flag is off")) + } +}