feat(#227): add filecategory field

api/openapi-v3-spec.json

@@ -2612,6 +2612,9 @@
           "configuration": {
             "$ref": "#/components/schemas/RepositoryConfigurationDto"
           },
+          "filecategory": {
+            "$ref": "#/components/schemas/RepositoryFileCategoriesDto"
+          },
           "timeStamp": {
             "type": "string",
             "description": "ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates.",
@@ -2662,6 +2665,9 @@
           "configuration": {
             "$ref": "#/components/schemas/RepositoryConfigurationDto"
           },
+          "filecategory": {
+            "$ref": "#/components/schemas/RepositoryFileCategoriesDto"
+          },
           "jiraIssue": {
             "type": "string",
             "description": "The jira issue to use for committing a change, or the last jira issue used.",
@@ -2701,6 +2707,9 @@
           "configuration": {
             "$ref": "#/components/schemas/RepositoryConfigurationDto"
           },
+          "filecategory": {
+            "$ref": "#/components/schemas/RepositoryFileCategoriesDto"
+          },
           "timeStamp": {
             "type": "string",
             "description": "ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates.",
@@ -2906,6 +2915,19 @@
           }
         }
       },
+      "RepositoryFileCategoriesDto": {
+        "type": "object",
+        "description": "Assign a category to a list of file globs, e.g. to mark them for caching purposes. The key is the category name, and the value is a list of globs. Files are considered to have that category if their path matches any of the given globs.",
+        "additionalProperties": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "example": {
+          "cache-template": ["templates/*.yaml", "another/path/*/*.json"]
+        }
+      },
       "ConditionReferenceDto": {
         "type": "object",
         "description": "Configuration of conditional build references.",

internal/acorn/config/customconfigint.go

@@ -63,6 +63,8 @@ type CustomConfiguration interface {
 	RepositoryKeySeparator() string
 
 	NotificationConsumerConfigs() map[string]NotificationConsumerConfig
+
+	AllowedFileCategories() []string
 }
 
 type NotificationConsumerConfig struct {
@@ -117,4 +119,5 @@ const (
 	KeyRepositoryKeySeparator         = "REPOSITORY_KEY_SEPARATOR"
 	KeyRepositoryTypes                = "REPOSITORY_TYPES"
 	KeyNotificationConsumerConfigs    = "NOTIFICATION_CONSUMER_CONFIGS"
+	KeyAllowedFileCategories          = "ALLOWED_FILE_CATEGORIES"
 )

internal/repository/config/accessors.go

@@ -173,3 +173,7 @@ func (c *CustomConfigImpl) RepositoryKeySeparator() string {
 func (c *CustomConfigImpl) NotificationConsumerConfigs() map[string]config.NotificationConsumerConfig {
 	return c.VNotificationConsumerConfigs
 }
+
+func (c *CustomConfigImpl) AllowedFileCategories() []string {
+	return c.VAllowedFileCategories
+}

internal/repository/config/config.go

@@ -297,4 +297,15 @@ var CustomConfigItems = []auconfigapi.ConfigItem{
 			return err
 		},
 	},
+	{
+		Key:         config.KeyAllowedFileCategories,
+		EnvName:     config.KeyAllowedFileCategories,
+		Default:     "",
+		Description: "allowed filecategory keys",
+		Validate: func(key string) error {
+			value := auconfigenv.Get(key)
+			_, err := parseAllowedFileCategories(value)
+			return err
+		},
+	},
 }

internal/repository/config/plumbing.go

@@ -63,6 +63,7 @@ type CustomConfigImpl struct {
 	VRepositoryTypes                string
 	VRepositoryKeySeparator         string
 	VNotificationConsumerConfigs    map[string]config.NotificationConsumerConfig
+	VAllowedFileCategories          []string
 }
 
 func New() (librepo.Configuration, config.CustomConfiguration) {
@@ -119,6 +120,7 @@ func (c *CustomConfigImpl) Obtain(getter func(key string) string) {
 	c.VRepositoryTypes = getter(config.KeyRepositoryTypes)
 	c.VRepositoryKeySeparator = getter(config.KeyRepositoryKeySeparator)
 	c.VNotificationConsumerConfigs, _ = parseNotificationConsumerConfigs(getter(config.KeyNotificationConsumerConfigs))
+	c.VAllowedFileCategories, _ = parseAllowedFileCategories(getter(config.KeyAllowedFileCategories))
 }
 
 // used after validation, so known safe
@@ -212,3 +214,16 @@ func parseNotificationConsumerConfigs(rawJson string) (map[string]config.Notific
 	}
 	return result, nil
 }
+
+func parseAllowedFileCategories(rawJson string) ([]string, error) {
+	result := make([]string, 0)
+	if rawJson == "" {
+		return result, nil
+	}
+
+	if err := json.Unmarshal([]byte(rawJson), &result); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}

internal/repository/config/validation_test.go

@@ -78,7 +78,7 @@ func TestValidate_LotsOfErrors(t *testing.T) {
 	_, err := tstSetupCutAndLogRecorder(t, "invalid-config-values.yaml")
 
 	require.NotNil(t, err)
-	require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 28 error(s). See details above")
+	require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 29 error(s). See details above")
 
 	actualLog := goauzerolog.RecordedLogForTesting.String()
 

internal/service/repositories/repositories.go

@@ -224,6 +224,7 @@ func (s *Impl) mapRepoCreateDtoToRepoDto(repositoryCreateDto openapi.RepositoryC
 		Url:           repositoryCreateDto.Url,
 		Mainline:      repositoryCreateDto.Mainline,
 		Configuration: repositoryCreateDto.Configuration,
+		Filecategory:  repositoryCreateDto.Filecategory,
 		Generator:     repositoryCreateDto.Generator,
 		Unittest:      repositoryCreateDto.Unittest,
 	}
@@ -395,6 +396,7 @@ func patchRepository(current openapi.RepositoryDto, patch openapi.RepositoryPatc
 		Generator:     patchStringPtr(patch.Generator, current.Generator),
 		Unittest:      patchPtr[bool](patch.Unittest, current.Unittest),
 		Configuration: patchConfiguration(patch.Configuration, current.Configuration),
+		Filecategory:  patchFilecategory(patch.Filecategory, current.Filecategory),
 		TimeStamp:     patch.TimeStamp,
 		CommitHash:    patch.CommitHash,
 		JiraIssue:     patch.JiraIssue,
@@ -437,6 +439,14 @@ func patchConditions(patch *map[string]openapi.ConditionReferenceDto, original *
 }
 
 func patchApprovers(patch *map[string][]string, original *map[string][]string) *map[string][]string {
+	return patchMapStringListString(patch, original)
+}
+
+func patchFilecategory(patch *map[string][]string, original *map[string][]string) *map[string][]string {
+	return patchMapStringListString(patch, original)
+}
+
+func patchMapStringListString(patch *map[string][]string, original *map[string][]string) *map[string][]string {
 	if patch != nil {
 		if len(*patch) == 0 {
 			// remove

internal/service/repositories/repositories_test.go

@@ -59,8 +59,9 @@ func createRepositoryDto() openapi.RepositoryDto {
 			DefaultReviewers: []string{"defaultreviewer1"},
 			SignedApprovers:  []string{"signedapprover1"},
 		},
-		TimeStamp:  "ts",
-		CommitHash: "hash",
+		Filecategory: &map[string][]string{"a": {"path/*.yaml"}},
+		TimeStamp:    "ts",
+		CommitHash:   "hash",
 	}
 }
 
@@ -134,8 +135,9 @@ func TestPatchRepository_ReplaceAll(t *testing.T) {
 			DefaultReviewers: []string{"newdefaultreviewer1"},
 			SignedApprovers:  []string{"newsignedapprover1"},
 		},
-		TimeStamp:  "newts",
-		CommitHash: "newhash",
+		Filecategory: &map[string][]string{"b": {"*.yaml", "*.json"}},
+		TimeStamp:    "newts",
+		CommitHash:   "newhash",
 	}, openapi.RepositoryDto{
 		Owner:     "newowner",
 		Url:       "newurl",
@@ -167,8 +169,9 @@ func TestPatchRepository_ReplaceAll(t *testing.T) {
 			DefaultReviewers: []string{"newdefaultreviewer1"},
 			SignedApprovers:  []string{"newsignedapprover1"},
 		},
-		TimeStamp:  "newts",
-		CommitHash: "newhash",
+		Filecategory: &map[string][]string{"b": {"*.yaml", "*.json"}},
+		TimeStamp:    "newts",
+		CommitHash:   "newhash",
 	})
 }
 
@@ -189,8 +192,9 @@ func TestPatchRepository_ClearFields(t *testing.T) {
 			DefaultReviewers: []string{},
 			SignedApprovers:  []string{},
 		},
-		TimeStamp:  "",
-		CommitHash: "",
+		Filecategory: &map[string][]string{},
+		TimeStamp:    "",
+		CommitHash:   "",
 	}, openapi.RepositoryDto{
 		Owner:     "",
 		Url:       "",
@@ -210,8 +214,9 @@ func TestPatchRepository_ClearFields(t *testing.T) {
 			DefaultReviewers: nil,
 			SignedApprovers:  nil,
 		},
-		TimeStamp:  "",
-		CommitHash: "",
+		Filecategory: nil,
+		TimeStamp:    "",
+		CommitHash:   "",
 	})
 }
 

local-config.template.yaml

@@ -45,6 +45,8 @@ ALERT_TARGET_SUFFIX: '@domain.com'
 
 OWNER_ALIAS_FILTER_REGEX: '.*'
 
+ALLOWED_FILE_CATEGORIES: '["template"]'
+
 # The NOTIFICATION_CONSUMER_CONFIGS env below is an example:
 
 #NOTIFICATION_CONSUMER_CONFIGS: >-

test/acceptance/util_dtos_test.go

@@ -223,6 +223,9 @@ func tstUpdatedServicePayload(name string) openapi.NotificationPayload {
 // repository
 
 func tstRepository() openapi.RepositoryDto {
+	fc := map[string][]string{
+		"cached-template": {"cached-templates/*.yaml", "more/cached/templates/*.yaml"},
+	}
 	return openapi.RepositoryDto{
 		Owner:    "some-owner",
 		Url:      "ssh://[email protected]:7999/helm/karma-wrapper.git",
@@ -251,9 +254,10 @@ func tstRepository() openapi.RepositoryDto {
 			},
 			Approvers: &map[string][]string{"testing": {"some-user"}},
 		},
-		TimeStamp:  "2022-11-06T18:14:10Z",
-		CommitHash: "6c8ac2c35791edf9979623c717a243fc53400000",
-		JiraIssue:  "ISSUE-2345",
+		Filecategory: &fc,
+		TimeStamp:    "2022-11-06T18:14:10Z",
+		CommitHash:   "6c8ac2c35791edf9979623c717a243fc53400000",
+		JiraIssue:    "ISSUE-2345",
 	}
 }
 
@@ -310,6 +314,10 @@ configuration:
     approvers:
         testing:
             - some-user
+filecategory:
+    cached-template:
+        - cached-templates/*.yaml
+        - more/cached/templates/*.yaml
 `
 }
 

test/acceptance/util_notifier_test.go

@@ -17,6 +17,5 @@ func hasSentNotification(t *testing.T, clientIdentifier string, name string, eve
 		Type:    payloadType.String(),
 		Payload: payload,
 	}
-	notifications := mockClient.SentNotifications
-	require.Contains(t, notifications, expected)
+	require.Contains(t, mockClient.SentNotifications, mockClient.ToJson(expected))
 }

test/acceptance/util_setup_test.go

@@ -2,7 +2,6 @@ package acceptance
 
 import (
 	"context"
-	"github.com/Interhyp/metadata-service/api"
 	"github.com/Interhyp/metadata-service/internal/repository/config"
 	"github.com/Interhyp/metadata-service/internal/repository/notifier"
 	"github.com/Interhyp/metadata-service/internal/service/trigger"
@@ -122,7 +121,7 @@ func tstSetup(configPath string) error {
 	security.Now = fakeNow
 
 	for identifier, _ := range notifierImpl.Clients {
-		notifierImpl.Clients[identifier] = &notifiermock.NotifierClientMock{SentNotifications: make([]openapi.Notification, 0)}
+		notifierImpl.Clients[identifier] = &notifiermock.NotifierClientMock{SentNotifications: make([]string, 0)}
 	}
 
 	tstSetupHttpTestServer()

test/mock/configmock/configmock.go

@@ -257,3 +257,8 @@ func (c *MockConfig) NotificationConsumerConfigs() map[string]config.Notificatio
 	//TODO implement me
 	panic("implement me")
 }
+
+func (c *MockConfig) AllowedFileCategories() []string {
+	//TODO implement me
+	panic("implement me")
+}

test/mock/notifiermock/notifierclientmock.go

@@ -2,21 +2,31 @@ package notifiermock
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	openapi "github.com/Interhyp/metadata-service/api"
 )
 
 type NotifierClientMock struct {
-	SentNotifications []openapi.Notification
+	SentNotifications []string
 }
 
 func (n *NotifierClientMock) Setup(clientIdentifier string, url string) error {
 	return nil
 }
 
 func (n *NotifierClientMock) Send(ctx context.Context, notification openapi.Notification) {
-	n.SentNotifications = append(n.SentNotifications, notification)
+	n.SentNotifications = append(n.SentNotifications, n.ToJson(notification))
 }
 
 func (n *NotifierClientMock) Reset() {
-	n.SentNotifications = make([]openapi.Notification, 0)
+	n.SentNotifications = make([]string, 0)
+}
+
+func (n *NotifierClientMock) ToJson(notification openapi.Notification) string {
+	notificationJson, err := json.Marshal(&notification)
+	if err != nil {
+		notificationJson = []byte(fmt.Sprintf("error: %s", err.Error()))
+	}
+	return string(notificationJson)
 }

test/resources/acceptance-expected/repository-create.json

@@ -29,6 +29,12 @@
       "pipelineTrigger": false
     }
   },
+  "filecategory": {
+    "cached-template": [
+      "cached-templates/*.yaml",
+      "more/cached/templates/*.yaml"
+    ]
+  },
   "jiraIssue": "ISSUE-2345",
   "mainline": "master",
   "owner": "some-owner",