Merge pull request #66 from port-labs/PORT-8146-filter-events-without…

…-diff

PORT-8146 filter events without diff in port

pkg/config/config.go

@@ -39,6 +39,7 @@ func Init() {
 	NewString(&ApplicationConfig.PortClientSecret, "port-client-secret", "", "Port client secret. Required.")
 	NewBool(&ApplicationConfig.CreateDefaultResources, "create-default-resources", true, "Create default resources on installation. Optional.")
 	NewBool(&ApplicationConfig.OverwriteConfigurationOnRestart, "overwrite-configuration-on-restart", false, "Overwrite the configuration in port on restarting the exporter. Optional.")
+	NewBool(&ApplicationConfig.UpdateEntityOnlyOnDiff, "update-entity-only-on-diff", false, "Optimization to reduce requests to port. Optional.")
 
 	// Deprecated
 	NewBool(&ApplicationConfig.DeleteDependents, "delete-dependents", false, "Delete dependents. Optional.")

pkg/config/models.go

@@ -26,4 +26,5 @@ type ApplicationConfiguration struct {
 	Resources                    []port.Resource
 	DeleteDependents             bool `json:"deleteDependents,omitempty"`
 	CreateMissingRelatedEntities bool `json:"createMissingRelatedEntities,omitempty"`
+	UpdateEntityOnlyOnDiff       bool `json:"updateEntityOnlyOnDiff,omitempty"`
 }

pkg/k8s/controller.go

@@ -10,11 +10,14 @@ import (
 	"github.com/port-labs/port-k8s-exporter/pkg/port"
 	"github.com/port-labs/port-k8s-exporter/pkg/port/cli"
 	"github.com/port-labs/port-k8s-exporter/pkg/port/mapping"
+	"hash/fnv"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/informers"
+	"strconv"
 
+	"encoding/json"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
@@ -69,7 +72,10 @@ func NewController(resource port.AggregatedResource, portClient *cli.PortClient,
 			item.ActionType = UpdateAction
 			item.Key, err = cache.MetaNamespaceKeyFunc(new)
 			if err == nil {
-				controller.workqueue.Add(item)
+
+				if controller.shouldSendUpdateEvent(old, new, config.ApplicationConfig.UpdateEntityOnlyOnDiff) {
+					controller.workqueue.Add(item)
+				}
 			}
 		},
 		DeleteFunc: func(obj interface{}) {
@@ -246,7 +252,7 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector,
 	entities := make([]port.Entity, 0, len(mappings))
 	objectsToMap := make([]interface{}, 0)
 
-	if (itemsToParse == "") {
+	if itemsToParse == "" {
 		objectsToMap = append(objectsToMap, structuredObj)
 	} else {
 		items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj)
@@ -268,7 +274,7 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector,
 			objectsToMap = append(objectsToMap, copiedObject)
 		}
 	}
-	
+
 	for _, objectToMap := range objectsToMap {
 		selectorResult, err := isPassSelector(objectToMap, selector)
 
@@ -384,3 +390,53 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) {
 
 	return k8sEntitiesSet, nil
 }
+
+func hashAllEntities(entities []port.Entity) (string, error) {
+	h := fnv.New64a()
+	for _, entity := range entities {
+		entityBytes, err := json.Marshal(entity)
+		if err != nil {
+			return "", err
+		}
+		_, err = h.Write(entityBytes)
+		if err != nil {
+			return "", err
+		}
+	}
+	return strconv.FormatUint(h.Sum64(), 10), nil
+}
+
+func (c *Controller) shouldSendUpdateEvent(old interface{}, new interface{}, updateEntityOnlyOnDiff bool) bool {
+
+	if updateEntityOnlyOnDiff == false {
+		return true
+	}
+	for _, kindConfig := range c.resource.KindConfigs {
+		oldEntities, err := c.getObjectEntities(old, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse)
+		if err != nil {
+			klog.Errorf("Error getting old entities: %v", err)
+			return true
+		}
+		newEntities, err := c.getObjectEntities(new, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse)
+		if err != nil {
+			klog.Errorf("Error getting new entities: %v", err)
+			return true
+		}
+		oldEntitiesHash, err := hashAllEntities(oldEntities)
+		if err != nil {
+			klog.Errorf("Error hashing old entities: %v", err)
+			return true
+		}
+		newEntitiesHash, err := hashAllEntities(newEntities)
+		if err != nil {
+			klog.Errorf("Error hashing new entities: %v", err)
+			return true
+		}
+
+		if oldEntitiesHash != newEntitiesHash {
+			return true
+		}
+	}
+
+	return false
+}

pkg/k8s/controller_test.go

@@ -3,6 +3,7 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"github.com/stretchr/testify/assert"
 	"reflect"
 	"strings"
 	"testing"
@@ -19,6 +20,7 @@ import (
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/dynamic/dynamicinformer"
 	"k8s.io/client-go/tools/cache"
@@ -102,6 +104,40 @@ func newDeployment() *appsv1.Deployment {
 	}
 }
 
+func newDeploymentWithCustomLabels(generation int64,
+	generateName string,
+	creationTimestamp v1.Time,
+	labels map[string]string,
+) *appsv1.Deployment {
+	return &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              "port-k8s-exporter",
+			Namespace:         "port-k8s-exporter",
+			GenerateName:      generateName,
+			Generation:        generation,
+			CreationTimestamp: creationTimestamp,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "port-k8s-exporter",
+							Image: "port-k8s-exporter:latest",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func newUnstructured(obj interface{}) *unstructured.Unstructured {
 	res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
 	if err != nil {
@@ -317,3 +353,97 @@ func TestGetEntitiesSet(t *testing.T) {
 	f := newFixture(t, "", "", "", resource, objects)
 	f.runControllerGetEntitiesSet(expectedEntitiesSet, false)
 }
+
+func TestUpdateHandlerWithIndividualPropertyChanges(t *testing.T) {
+
+	type Property struct {
+		Value           interface{}
+		ShouldSendEvent bool
+	}
+
+	fullMapping := []port.Resource{
+		newResource("", []port.EntityMapping{
+			{
+				Identifier: ".metadata.name",
+				Blueprint:  "\"k8s-export-test-bp\"",
+				Icon:       "\"Microservice\"",
+				Team:       "\"Test\"",
+				Properties: map[string]string{
+					"labels":            ".spec.selector",
+					"generation":        ".metadata.generation",
+					"generateName":      ".metadata.generateName",
+					"creationTimestamp": ".metadata.creationTimestamp",
+				},
+				Relations: map[string]string{
+					"k8s-relation": "\"e_AgPMYvq1tAs8TuqM\"",
+				},
+			},
+		}),
+		newResource("", []port.EntityMapping{
+
+			{
+				Identifier: ".metadata.name",
+				Blueprint:  "\"k8s-export-test-bp\"",
+				Icon:       "\"Microservice\"",
+				Team:       "\"Test\"",
+				Properties: map[string]string{},
+				Relations:  map[string]string{},
+			},
+			{
+				Identifier: ".metadata.name",
+				Blueprint:  "\"k8s-export-test-bp\"",
+				Icon:       "\"Microservice\"",
+				Team:       "\"Test\"",
+				Properties: map[string]string{
+					"labels":            ".spec.selector",
+					"generation":        ".metadata.generation",
+					"generateName":      ".metadata.generateName",
+					"creationTimestamp": ".metadata.creationTimestamp",
+				},
+				Relations: map[string]string{
+					"k8s-relation": "\"e_AgPMYvq1tAs8TuqM\"",
+				},
+			},
+		}),
+	}
+
+	for _, mapping := range fullMapping {
+
+		controllerWithFullMapping := newFixture(t, "", "", "", mapping, []runtime.Object{}).controller
+
+		// Test changes in each individual property
+		properties := map[string]Property{
+			"metadata.name":              {Value: "port-k8s-exporter", ShouldSendEvent: false},
+			"something_without_mapping":  {Value: "port-k8s-exporter", ShouldSendEvent: false},
+			"metadata.generation":        {Value: int64(3), ShouldSendEvent: true},
+			"metadata.generateName":      {Value: "new-port-k8s-exporter2", ShouldSendEvent: true},
+			"metadata.creationTimestamp": {Value: v1.Now().Add(1 * time.Hour).Format(time.RFC3339), ShouldSendEvent: true},
+		}
+
+		for property, value := range properties {
+			newDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"}))
+			oldDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"}))
+
+			// Update the property in the new deployment
+			unstructured.SetNestedField(newDep.Object, value.Value, strings.Split(property, ".")...)
+
+			result := controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, true)
+			if value.ShouldSendEvent {
+				assert.True(t, result, fmt.Sprintf("Expected true when %s changes and feature flag is on", property))
+			} else {
+				assert.False(t, result, fmt.Sprintf("Expected false when %s changes and feature flag is on", property))
+			}
+			result = controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, false)
+			assert.True(t, result, fmt.Sprintf("Expected true when %s changes and feature flag is off", property))
+		}
+
+		// Add a case for json update because you can't edit the json directly
+		newDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "port-k8s-exporter"}))
+		oldDep := newUnstructured(newDeploymentWithCustomLabels(2, "new-port-k8s-exporter", v1.Now(), map[string]string{"app": "new-port-k8s-exporter"}))
+
+		result := controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, true)
+		assert.True(t, result, fmt.Sprintf("Expected true when labels changes and feature flag is on"))
+		result = controllerWithFullMapping.shouldSendUpdateEvent(oldDep, newDep, false)
+		assert.True(t, result, fmt.Sprintf("Expected true when labels changes and feature flag is off"))
+	}
+}