diff --git a/api/generated_apimodel.go b/api/generated_apimodel.go index 0d08fa3..fd8e232 100644 --- a/api/generated_apimodel.go +++ b/api/generated_apimodel.go @@ -2,13 +2,7 @@ package openapi -import ( - "time" -) - -type _dummyTime struct { - Timestamp *time.Time -} +import "time" type ConditionReferenceDto struct { // Reference of a branch. @@ -179,6 +173,8 @@ type RepositoryCreateDto struct { // this repository contains unit tests (currently ignored except for helm charts) Unittest *bool `yaml:"unittest,omitempty" json:"unittest,omitempty"` Configuration *RepositoryConfigurationDto `yaml:"configuration,omitempty" json:"configuration,omitempty"` + // Assign a category to a list of file globs, e.g. to mark them for caching purposes. The key is the category name, and the value is a list of globs. Files are considered to have that category if their path matches any of the given globs. + Filecategory *map[string][]string `yaml:"filecategory,omitempty" json:"filecategory,omitempty"` // The jira issue to use for committing a change, or the last jira issue used. JiraIssue string `yaml:"-" json:"jiraIssue"` } @@ -193,6 +189,8 @@ type RepositoryDto struct { // this repository contains unit tests (currently ignored except for helm charts) Unittest *bool `yaml:"unittest,omitempty" json:"unittest,omitempty"` Configuration *RepositoryConfigurationDto `yaml:"configuration,omitempty" json:"configuration,omitempty"` + // Assign a category to a list of file globs, e.g. to mark them for caching purposes. The key is the category name, and the value is a list of globs. Files are considered to have that category if their path matches any of the given globs. + Filecategory *map[string][]string `yaml:"filecategory,omitempty" json:"filecategory,omitempty"` // ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates. TimeStamp string `yaml:"-" json:"timeStamp"` // The git commit hash this information was originally committed under. When sending an update, include the original commitHash you got so we can detect concurrent updates. @@ -217,6 +215,8 @@ type RepositoryPatchDto struct { // this repository contains unit tests (currently ignored except for helm charts) Unittest *bool `yaml:"unittest,omitempty" json:"unittest,omitempty"` Configuration *RepositoryConfigurationDto `yaml:"configuration,omitempty" json:"configuration,omitempty"` + // Assign a category to a list of file globs, e.g. to mark them for caching purposes. The key is the category name, and the value is a list of globs. Files are considered to have that category if their path matches any of the given globs. + Filecategory *map[string][]string `yaml:"filecategory,omitempty" json:"filecategory,omitempty"` // ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates. TimeStamp string `yaml:"-" json:"timeStamp"` // The git commit hash this information was originally committed under. When sending an update, include the original commitHash you got so we can detect concurrent updates. diff --git a/api/openapi-v3-spec.json b/api/openapi-v3-spec.json index 68ca414..2b8f300 100644 --- a/api/openapi-v3-spec.json +++ b/api/openapi-v3-spec.json @@ -2612,6 +2612,9 @@ "configuration": { "$ref": "#/components/schemas/RepositoryConfigurationDto" }, + "filecategory": { + "$ref": "#/components/schemas/RepositoryFileCategoriesDto" + }, "timeStamp": { "type": "string", "description": "ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates.", @@ -2662,6 +2665,9 @@ "configuration": { "$ref": "#/components/schemas/RepositoryConfigurationDto" }, + "filecategory": { + "$ref": "#/components/schemas/RepositoryFileCategoriesDto" + }, "jiraIssue": { "type": "string", "description": "The jira issue to use for committing a change, or the last jira issue used.", @@ -2701,6 +2707,9 @@ "configuration": { "$ref": "#/components/schemas/RepositoryConfigurationDto" }, + "filecategory": { + "$ref": "#/components/schemas/RepositoryFileCategoriesDto" + }, "timeStamp": { "type": "string", "description": "ISO-8601 UTC date time at which this information was originally committed. When sending an update, include the original timestamp you got so we can detect concurrent updates.", @@ -2906,6 +2915,19 @@ } } }, + "RepositoryFileCategoriesDto": { + "type": "object", + "description": "Assign a category to a list of file globs, e.g. to mark them for caching purposes. The key is the category name, and the value is a list of globs. Files are considered to have that category if their path matches any of the given globs.", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + }, + "example": { + "cache-template": ["templates/*.yaml", "another/path/*/*.json"] + } + }, "ConditionReferenceDto": { "type": "object", "description": "Configuration of conditional build references.", diff --git a/internal/acorn/config/customconfigint.go b/internal/acorn/config/customconfigint.go index 8c45110..b5c6378 100644 --- a/internal/acorn/config/customconfigint.go +++ b/internal/acorn/config/customconfigint.go @@ -63,6 +63,8 @@ type CustomConfiguration interface { RepositoryKeySeparator() string NotificationConsumerConfigs() map[string]NotificationConsumerConfig + + AllowedFileCategories() []string } type NotificationConsumerConfig struct { @@ -117,4 +119,5 @@ const ( KeyRepositoryKeySeparator = "REPOSITORY_KEY_SEPARATOR" KeyRepositoryTypes = "REPOSITORY_TYPES" KeyNotificationConsumerConfigs = "NOTIFICATION_CONSUMER_CONFIGS" + KeyAllowedFileCategories = "ALLOWED_FILE_CATEGORIES" ) diff --git a/internal/repository/config/accessors.go b/internal/repository/config/accessors.go index bc27b5a..374c06a 100644 --- a/internal/repository/config/accessors.go +++ b/internal/repository/config/accessors.go @@ -173,3 +173,7 @@ func (c *CustomConfigImpl) RepositoryKeySeparator() string { func (c *CustomConfigImpl) NotificationConsumerConfigs() map[string]config.NotificationConsumerConfig { return c.VNotificationConsumerConfigs } + +func (c *CustomConfigImpl) AllowedFileCategories() []string { + return c.VAllowedFileCategories +} diff --git a/internal/repository/config/config.go b/internal/repository/config/config.go index f9ec2f2..4ccc805 100644 --- a/internal/repository/config/config.go +++ b/internal/repository/config/config.go @@ -297,4 +297,15 @@ var CustomConfigItems = []auconfigapi.ConfigItem{ return err }, }, + { + Key: config.KeyAllowedFileCategories, + EnvName: config.KeyAllowedFileCategories, + Default: "", + Description: "allowed filecategory keys", + Validate: func(key string) error { + value := auconfigenv.Get(key) + _, err := parseAllowedFileCategories(value) + return err + }, + }, } diff --git a/internal/repository/config/plumbing.go b/internal/repository/config/plumbing.go index 6983ba5..87ec1f3 100644 --- a/internal/repository/config/plumbing.go +++ b/internal/repository/config/plumbing.go @@ -63,6 +63,7 @@ type CustomConfigImpl struct { VRepositoryTypes string VRepositoryKeySeparator string VNotificationConsumerConfigs map[string]config.NotificationConsumerConfig + VAllowedFileCategories []string } func New() (librepo.Configuration, config.CustomConfiguration) { @@ -119,6 +120,7 @@ func (c *CustomConfigImpl) Obtain(getter func(key string) string) { c.VRepositoryTypes = getter(config.KeyRepositoryTypes) c.VRepositoryKeySeparator = getter(config.KeyRepositoryKeySeparator) c.VNotificationConsumerConfigs, _ = parseNotificationConsumerConfigs(getter(config.KeyNotificationConsumerConfigs)) + c.VAllowedFileCategories, _ = parseAllowedFileCategories(getter(config.KeyAllowedFileCategories)) } // used after validation, so known safe @@ -212,3 +214,16 @@ func parseNotificationConsumerConfigs(rawJson string) (map[string]config.Notific } return result, nil } + +func parseAllowedFileCategories(rawJson string) ([]string, error) { + result := make([]string, 0) + if rawJson == "" { + return result, nil + } + + if err := json.Unmarshal([]byte(rawJson), &result); err != nil { + return nil, err + } + + return result, nil +} diff --git a/internal/repository/config/validation_test.go b/internal/repository/config/validation_test.go index 5cf3948..6cfeb3d 100644 --- a/internal/repository/config/validation_test.go +++ b/internal/repository/config/validation_test.go @@ -78,7 +78,7 @@ func TestValidate_LotsOfErrors(t *testing.T) { _, err := tstSetupCutAndLogRecorder(t, "invalid-config-values.yaml") require.NotNil(t, err) - require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 28 error(s). See details above") + require.Contains(t, err.Error(), "some configuration values failed to validate or parse. There were 29 error(s). See details above") actualLog := goauzerolog.RecordedLogForTesting.String() diff --git a/internal/service/repositories/repositories.go b/internal/service/repositories/repositories.go index 5353730..2b2e922 100644 --- a/internal/service/repositories/repositories.go +++ b/internal/service/repositories/repositories.go @@ -224,6 +224,7 @@ func (s *Impl) mapRepoCreateDtoToRepoDto(repositoryCreateDto openapi.RepositoryC Url: repositoryCreateDto.Url, Mainline: repositoryCreateDto.Mainline, Configuration: repositoryCreateDto.Configuration, + Filecategory: repositoryCreateDto.Filecategory, Generator: repositoryCreateDto.Generator, Unittest: repositoryCreateDto.Unittest, } @@ -395,6 +396,7 @@ func patchRepository(current openapi.RepositoryDto, patch openapi.RepositoryPatc Generator: patchStringPtr(patch.Generator, current.Generator), Unittest: patchPtr[bool](patch.Unittest, current.Unittest), Configuration: patchConfiguration(patch.Configuration, current.Configuration), + Filecategory: patchFilecategory(patch.Filecategory, current.Filecategory), TimeStamp: patch.TimeStamp, CommitHash: patch.CommitHash, JiraIssue: patch.JiraIssue, @@ -437,6 +439,14 @@ func patchConditions(patch *map[string]openapi.ConditionReferenceDto, original * } func patchApprovers(patch *map[string][]string, original *map[string][]string) *map[string][]string { + return patchMapStringListString(patch, original) +} + +func patchFilecategory(patch *map[string][]string, original *map[string][]string) *map[string][]string { + return patchMapStringListString(patch, original) +} + +func patchMapStringListString(patch *map[string][]string, original *map[string][]string) *map[string][]string { if patch != nil { if len(*patch) == 0 { // remove diff --git a/internal/service/repositories/repositories_test.go b/internal/service/repositories/repositories_test.go index 167f3c7..5f88f55 100644 --- a/internal/service/repositories/repositories_test.go +++ b/internal/service/repositories/repositories_test.go @@ -59,8 +59,9 @@ func createRepositoryDto() openapi.RepositoryDto { DefaultReviewers: []string{"defaultreviewer1"}, SignedApprovers: []string{"signedapprover1"}, }, - TimeStamp: "ts", - CommitHash: "hash", + Filecategory: &map[string][]string{"a": {"path/*.yaml"}}, + TimeStamp: "ts", + CommitHash: "hash", } } @@ -134,8 +135,9 @@ func TestPatchRepository_ReplaceAll(t *testing.T) { DefaultReviewers: []string{"newdefaultreviewer1"}, SignedApprovers: []string{"newsignedapprover1"}, }, - TimeStamp: "newts", - CommitHash: "newhash", + Filecategory: &map[string][]string{"b": {"*.yaml", "*.json"}}, + TimeStamp: "newts", + CommitHash: "newhash", }, openapi.RepositoryDto{ Owner: "newowner", Url: "newurl", @@ -167,8 +169,9 @@ func TestPatchRepository_ReplaceAll(t *testing.T) { DefaultReviewers: []string{"newdefaultreviewer1"}, SignedApprovers: []string{"newsignedapprover1"}, }, - TimeStamp: "newts", - CommitHash: "newhash", + Filecategory: &map[string][]string{"b": {"*.yaml", "*.json"}}, + TimeStamp: "newts", + CommitHash: "newhash", }) } @@ -189,8 +192,9 @@ func TestPatchRepository_ClearFields(t *testing.T) { DefaultReviewers: []string{}, SignedApprovers: []string{}, }, - TimeStamp: "", - CommitHash: "", + Filecategory: &map[string][]string{}, + TimeStamp: "", + CommitHash: "", }, openapi.RepositoryDto{ Owner: "", Url: "", @@ -210,8 +214,9 @@ func TestPatchRepository_ClearFields(t *testing.T) { DefaultReviewers: nil, SignedApprovers: nil, }, - TimeStamp: "", - CommitHash: "", + Filecategory: nil, + TimeStamp: "", + CommitHash: "", }) } diff --git a/local-config.template.yaml b/local-config.template.yaml index 22223ec..60911ba 100644 --- a/local-config.template.yaml +++ b/local-config.template.yaml @@ -45,6 +45,8 @@ ALERT_TARGET_SUFFIX: '@domain.com' OWNER_ALIAS_FILTER_REGEX: '.*' +ALLOWED_FILE_CATEGORIES: '["template"]' + # The NOTIFICATION_CONSUMER_CONFIGS env below is an example: #NOTIFICATION_CONSUMER_CONFIGS: >- diff --git a/test/acceptance/util_dtos_test.go b/test/acceptance/util_dtos_test.go index dbe9ee6..d030cb2 100644 --- a/test/acceptance/util_dtos_test.go +++ b/test/acceptance/util_dtos_test.go @@ -223,6 +223,9 @@ func tstUpdatedServicePayload(name string) openapi.NotificationPayload { // repository func tstRepository() openapi.RepositoryDto { + fc := map[string][]string{ + "cached-template": {"cached-templates/*.yaml", "more/cached/templates/*.yaml"}, + } return openapi.RepositoryDto{ Owner: "some-owner", Url: "ssh://git@bitbucket.some-organisation.com:7999/helm/karma-wrapper.git", @@ -251,9 +254,10 @@ func tstRepository() openapi.RepositoryDto { }, Approvers: &map[string][]string{"testing": {"some-user"}}, }, - TimeStamp: "2022-11-06T18:14:10Z", - CommitHash: "6c8ac2c35791edf9979623c717a243fc53400000", - JiraIssue: "ISSUE-2345", + Filecategory: &fc, + TimeStamp: "2022-11-06T18:14:10Z", + CommitHash: "6c8ac2c35791edf9979623c717a243fc53400000", + JiraIssue: "ISSUE-2345", } } @@ -310,6 +314,10 @@ configuration: approvers: testing: - some-user +filecategory: + cached-template: + - cached-templates/*.yaml + - more/cached/templates/*.yaml ` } diff --git a/test/acceptance/util_notifier_test.go b/test/acceptance/util_notifier_test.go index ef91a8c..dc01e5f 100644 --- a/test/acceptance/util_notifier_test.go +++ b/test/acceptance/util_notifier_test.go @@ -17,6 +17,5 @@ func hasSentNotification(t *testing.T, clientIdentifier string, name string, eve Type: payloadType.String(), Payload: payload, } - notifications := mockClient.SentNotifications - require.Contains(t, notifications, expected) + require.Contains(t, mockClient.SentNotifications, mockClient.ToJson(expected)) } diff --git a/test/acceptance/util_setup_test.go b/test/acceptance/util_setup_test.go index 9da9b34..81927c4 100644 --- a/test/acceptance/util_setup_test.go +++ b/test/acceptance/util_setup_test.go @@ -2,7 +2,6 @@ package acceptance import ( "context" - "github.com/Interhyp/metadata-service/api" "github.com/Interhyp/metadata-service/internal/repository/config" "github.com/Interhyp/metadata-service/internal/repository/notifier" "github.com/Interhyp/metadata-service/internal/service/trigger" @@ -122,7 +121,7 @@ func tstSetup(configPath string) error { security.Now = fakeNow for identifier, _ := range notifierImpl.Clients { - notifierImpl.Clients[identifier] = ¬ifiermock.NotifierClientMock{SentNotifications: make([]openapi.Notification, 0)} + notifierImpl.Clients[identifier] = ¬ifiermock.NotifierClientMock{SentNotifications: make([]string, 0)} } tstSetupHttpTestServer() diff --git a/test/mock/configmock/configmock.go b/test/mock/configmock/configmock.go index 7f78e10..39c2901 100644 --- a/test/mock/configmock/configmock.go +++ b/test/mock/configmock/configmock.go @@ -257,3 +257,8 @@ func (c *MockConfig) NotificationConsumerConfigs() map[string]config.Notificatio //TODO implement me panic("implement me") } + +func (c *MockConfig) AllowedFileCategories() []string { + //TODO implement me + panic("implement me") +} diff --git a/test/mock/notifiermock/notifierclientmock.go b/test/mock/notifiermock/notifierclientmock.go index 8740dad..51475c7 100644 --- a/test/mock/notifiermock/notifierclientmock.go +++ b/test/mock/notifiermock/notifierclientmock.go @@ -2,11 +2,13 @@ package notifiermock import ( "context" + "encoding/json" + "fmt" openapi "github.com/Interhyp/metadata-service/api" ) type NotifierClientMock struct { - SentNotifications []openapi.Notification + SentNotifications []string } func (n *NotifierClientMock) Setup(clientIdentifier string, url string) error { @@ -14,9 +16,17 @@ func (n *NotifierClientMock) Setup(clientIdentifier string, url string) error { } func (n *NotifierClientMock) Send(ctx context.Context, notification openapi.Notification) { - n.SentNotifications = append(n.SentNotifications, notification) + n.SentNotifications = append(n.SentNotifications, n.ToJson(notification)) } func (n *NotifierClientMock) Reset() { - n.SentNotifications = make([]openapi.Notification, 0) + n.SentNotifications = make([]string, 0) +} + +func (n *NotifierClientMock) ToJson(notification openapi.Notification) string { + notificationJson, err := json.Marshal(¬ification) + if err != nil { + notificationJson = []byte(fmt.Sprintf("error: %s", err.Error())) + } + return string(notificationJson) } diff --git a/test/resources/acceptance-expected/repository-create.json b/test/resources/acceptance-expected/repository-create.json index 4c41a51..3a94e2f 100644 --- a/test/resources/acceptance-expected/repository-create.json +++ b/test/resources/acceptance-expected/repository-create.json @@ -29,6 +29,12 @@ "pipelineTrigger": false } }, + "filecategory": { + "cached-template": [ + "cached-templates/*.yaml", + "more/cached/templates/*.yaml" + ] + }, "jiraIssue": "ISSUE-2345", "mainline": "master", "owner": "some-owner", diff --git a/test/resources/acceptance-expected/repository-update-newowner.json b/test/resources/acceptance-expected/repository-update-newowner.json index 164407f..4980b0d 100644 --- a/test/resources/acceptance-expected/repository-update-newowner.json +++ b/test/resources/acceptance-expected/repository-update-newowner.json @@ -29,6 +29,12 @@ "pipelineTrigger": false } }, + "filecategory": { + "cached-template": [ + "cached-templates/*.yaml", + "more/cached/templates/*.yaml" + ] + }, "jiraIssue": "ISSUE-2345", "mainline": "master", "owner": "deleteme", diff --git a/test/resources/acceptance-expected/repository-update.json b/test/resources/acceptance-expected/repository-update.json index 4c41a51..3a94e2f 100644 --- a/test/resources/acceptance-expected/repository-update.json +++ b/test/resources/acceptance-expected/repository-update.json @@ -29,6 +29,12 @@ "pipelineTrigger": false } }, + "filecategory": { + "cached-template": [ + "cached-templates/*.yaml", + "more/cached/templates/*.yaml" + ] + }, "jiraIssue": "ISSUE-2345", "mainline": "master", "owner": "some-owner", diff --git a/test/resources/invalid-config-values.yaml b/test/resources/invalid-config-values.yaml index 6122cc9..4c85867 100644 --- a/test/resources/invalid-config-values.yaml +++ b/test/resources/invalid-config-values.yaml @@ -56,3 +56,5 @@ NOTIFICATION_CONSUMER_CONFIGS: >- "url": "https://valid.url.com/for/a/webhook" } } + +ALLOWED_FILE_CATEGORIES: '["a","b"' diff --git a/test/resources/valid-config-unique.yaml b/test/resources/valid-config-unique.yaml index 1e8e053..686c90f 100644 --- a/test/resources/valid-config-unique.yaml +++ b/test/resources/valid-config-unique.yaml @@ -47,4 +47,6 @@ REPOSITORY_NAME_MAX_LENGTH: '3' REPOSITORY_KEY_SEPARATOR: ';' REPOSITORY_TYPES: 'some-type,some-other-type' -NOTIFICATION_CONSUMER_CONFIGS: "{}" \ No newline at end of file +NOTIFICATION_CONSUMER_CONFIGS: "{}" + +ALLOWED_FILE_CATEGORIES: '["yaml-template"]' diff --git a/test/resources/valid-config.yaml b/test/resources/valid-config.yaml index 1194c0f..c8ea2d0 100644 --- a/test/resources/valid-config.yaml +++ b/test/resources/valid-config.yaml @@ -64,3 +64,5 @@ NOTIFICATION_CONSUMER_CONFIGS: >- "url": "https://some.url.com/for/the/webhook" } } + +ALLOWED_FILE_CATEGORIES: '["a","b"]'