From ff06b3d439c927cdf0f5a82b8f48ff1f75350fb0 Mon Sep 17 00:00:00 2001 From: Toby Bellwood Date: Wed, 18 Oct 2023 12:34:17 +1100 Subject: [PATCH 01/27] add additional transformers --- default_filter_transformers.yaml | 245 ++++++++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 5 deletions(-) diff --git a/default_filter_transformers.yaml b/default_filter_transformers.yaml index 360add3..12d4f4f 100644 --- a/default_filter_transformers.yaml +++ b/default_filter_transformers.yaml @@ -1,5 +1,184 @@ --- transforms: + +## Operating System transformers + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: alpine + exactMatch: true + transformations: + - name: Name + value: Alpine Linux + - name: Category + value: OS + - name: Description + value: Base image Alpine Linux version + keyfact: true + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: amzn + exactMatch: true + transformations: + - name: Name + value: Amazon Linux + - name: Category + value: OS + - name: Description + value: Base image Amazon Linux version + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: debian + exactMatch: true + transformations: + - name: Name + value: Debian Linux + - name: Category + value: OS + - name: Description + value: Base image Debian Linux version + +# Lagoon Transformers + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: LAGOON_VERSION + exactMatch: true + transformations: + - name: Name + value: Image Version + - name: Category + value: Lagoon + - name: Description + value: The currently running Lagoon Image version + keyfact: true + +# Service Transformers + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: mariadb-common + exactMatch: true + transformations: + - name: Name + value: MariaDB + - name: Category + value: Service + - name: Description + value: The currently running MariaDB service + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: mongodb + exactMatch: true + transformations: + - name: Name + value: MongoDB + - name: Category + value: Service + - name: Description + value: The currently running MongoDB service + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: nginx + exactMatch: true + transformations: + - name: Name + value: NGINX + - name: Category + value: Service + - name: Description + value: The currently running NGINX service + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: opensearch-core + exactMatch: true + transformations: + - name: Name + value: OpenSearch + - name: Category + value: Service + - name: Description + value: The currently running OpenSearch service + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: PG_VERSION + exactMatch: true + transformations: + - name: Name + value: PostgreSQL + - name: Category + value: Service + - name: Description + value: The currently running PostgreSQL service + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: RABBITMQ_VERSION + exactMatch: true + transformations: + - name: Name + value: RabbitMQ + - name: Category + value: Service + - name: Description + value: The currently running RabbitMQ service + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: REDIS_VERSION + exactMatch: true + transformations: + - name: Name + value: Redis + - name: Category + value: Service + - name: Description + value: The currently running Redis service + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: solr-core + exactMatch: true + transformations: + - name: Name + value: Solr + - name: Category + value: Service + - name: Description + value: The currently running Solr service + - type: cyclonedx.Component + lookupvalue: + - name: Name + value: varnish + exactMatch: true + transformations: + - name: Name + value: Varnish + - name: Category + value: Service + - name: Description + value: The currently running Varnish service + +# Language Transformers + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: NODE_VERSION + exactMatch: true + transformations: + - name: Name + value: NodeJS + - name: Category + value: Language + - name: Description + value: The currently installed NodeJS version + keyfact: true - type: handler.EnvironmentVariable lookupvalue: - name: Key @@ -13,6 +192,34 @@ transforms: - name: Description value: The currently installed PHP version keyfact: true + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: PYTHON_VERSION + exactMatch: true + transformations: + - name: Name + value: Python + - name: Category + value: Language + - name: Description + value: The currently installed Python version + keyfact: true + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: RUBY_VERSION + exactMatch: true + transformations: + - name: Name + value: Ruby + - name: Category + value: Language + - name: Description + value: The currently installed Ruby version + keyfact: true + +# Application Transformers - type: cyclonedx.Component lookupvalue: - name: Name @@ -27,16 +234,18 @@ transforms: - type: cyclonedx.Component lookupvalue: - name: Name - value: alpine + value: johnpbloch/wordpress-core exactMatch: true transformations: - name: Name - value: Alpine Linux + value: Wordpress - name: Category - value: OS + value: Application - name: Description - value: Base image Alpine Linux version + value: The currently installed Wordpress version keyfact: true + +# Helper Transformers - type: cyclonedx.Component lookupvalue: - name: Name @@ -46,4 +255,30 @@ transforms: value: Drush - name: Category value: Helper - keyfact: true \ No newline at end of file + keyfact: true + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: PYTHON_PIP_VERSION + exactMatch: true + transformations: + - name: Name + value: pip + - name: Category + value: Helper + - name: Description + value: The currently installed pip (Python package installer) version + keyfact: true + - type: handler.EnvironmentVariable + lookupvalue: + - name: Key + value: YARN_VERSION + exactMatch: true + transformations: + - name: Name + value: Yarn + - name: Category + value: Helper + - name: Description + value: The currently installed Yarn package manager version + keyfact: true From a52f99463631b0590172554555170204f550ab9a Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 24 Nov 2023 11:57:15 +1300 Subject: [PATCH 02/27] Adds error rets to message queue processing --- internal/handler/main.go | 45 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index db7bf2a..d8340ae 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -283,32 +283,43 @@ func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource Resource if insights.InputPayload == Payload && insights.LagoonType == Facts { for _, p := range incoming.Payload { - parserFilterLoopForPayloads(insights, p, h, apiClient, resource) + err := parserFilterLoopForPayloads(insights, p, h, apiClient, resource) + if err != nil { + return err + } } } if insights.InputPayload == BinaryPayload && insights.LagoonType == Facts { for _, p := range incoming.BinaryPayload { - parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) + err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) + if err != nil { + return err + } } } return nil } -func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) { +func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) error { for _, filter := range parserFilters { result, source, err := filter(h, insights, p, apiClient, resource) if err != nil { slog.Error("Error running filter", "error", err.Error()) + return err } - processResultset(result, err, h, apiClient, resource, source) + err = processResultset(result, err, h, apiClient, resource, source) + if err != nil { + return err + } } + return nil } -func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) { +func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) error { for _, filter := range parserFilters { var result []interface{} var source string @@ -316,22 +327,29 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa json, err := json.Marshal(p) if err != nil { slog.Error("Error marshalling data", "error", err.Error()) + return err } result, source, err = filter(h, insights, fmt.Sprintf("%s", json), apiClient, resource) if err != nil { slog.Error("Error Filtering payload", "error", err.Error()) + return err } - processResultset(result, err, h, apiClient, resource, source) + err = processResultset(result, err, h, apiClient, resource, source) + if err != nil { + return err + } } + return nil } // processResultset will send results as facts to the lagoon api after processing via a parser filter -func processResultset(result []interface{}, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) { +func processResultset(result []interface{}, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { project, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { log.Println(apiErr) + return apiErr } // Even if we don't find any new facts, we need to delete the existing ones @@ -339,6 +357,7 @@ func processResultset(result []interface{}, err error, h *Messaging, apiClient g apiErr = h.deleteExistingFactsBySource(apiClient, environment, source, project) if apiErr != nil { log.Printf("%s", apiErr.Error()) + return apiErr } for _, r := range result { @@ -347,15 +366,23 @@ func processResultset(result []interface{}, err error, h *Messaging, apiClient g err = h.sendFactsToLagoonAPI([]LagoonFact{fact}, apiClient, resource, source) if err != nil { slog.Error("Error sending facts to Lagoon API", "error", err.Error()) + return err } } else if facts, ok := r.([]LagoonFact); ok { // Handle slice of facts - h.sendFactsToLagoonAPI(facts, apiClient, resource, source) + err = h.sendFactsToLagoonAPI(facts, apiClient, resource, source) + if err != nil { + slog.Error("Error sending facts to Lagoon API", "error", err.Error()) + return err + } } else { // Unexpected type returned from filter() - slog.Error(fmt.Sprintf("unexpected type returned from filter(): %T\n", r)) + err := fmt.Errorf("unexpected type returned from filter(): %T\n", r) + slog.Error(err.Error()) + return err } } + return nil } func (h *Messaging) sendFactsToLagoonAPI(facts []LagoonFact, apiClient graphql.Client, resource ResourceDestination, source string) error { From 835f5a5596f22037f0bdc9c9b2889f8a255050cb Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 28 Nov 2023 06:20:31 +1300 Subject: [PATCH 03/27] Fixes log --- internal/handler/main.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index d8340ae..56d108e 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -217,13 +217,13 @@ func (h *Messaging) Consumer() { var err error messageQueue, err = mq.New(h.Config) if err != nil { - log.Println(err, - fmt.Sprintf( - "Failed to initialize message queue manager, retrying in %d seconds, attempt %d/%d", - h.ConnectionRetryInterval, - attempt, - h.ConnectionAttempts, - ), + slog.Error(fmt.Sprintf( + "Failed to initialize message queue manager, retrying in %d seconds, attempt %d/%d", + h.ConnectionRetryInterval, + attempt, + h.ConnectionAttempts, + ), + "error", err.Error(), ) time.Sleep(time.Duration(h.ConnectionRetryInterval) * time.Second) } From fc20bab24e4090a59bc1b3b25c163df7b3c1f7d1 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 28 Nov 2023 06:41:46 +1300 Subject: [PATCH 04/27] Fixes log --- internal/handler/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index 56d108e..b9db42b 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -348,7 +348,7 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa func processResultset(result []interface{}, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { project, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { - log.Println(apiErr) + slog.Error(apiErr.Error()) return apiErr } @@ -356,7 +356,7 @@ func processResultset(result []interface{}, err error, h *Messaging, apiClient g // since these may be the end product of a filter process apiErr = h.deleteExistingFactsBySource(apiClient, environment, source, project) if apiErr != nil { - log.Printf("%s", apiErr.Error()) + slog.Error(apiErr.Error()) return apiErr } From 3c7f819ee2f80a8662fd9744e19d6267976f63fb Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 4 Dec 2023 10:45:53 +1300 Subject: [PATCH 05/27] Removes extraneous comment --- internal/handler/messaging.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index f8582cb..88f9ffa 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -192,7 +192,6 @@ func (h *Messaging) processMessageQueue(message mq.Message) { err := h.sendToLagoonAPI(incoming, resource, insights) if err != nil { - //log.Printf("Unable to send to the api: %s", err.Error()) slog.Error("Unable to send to the API", "Error", err.Error()) rejectMessage(false) return From 81e567947a2b40aa5ef710354a5f9e41d26fa376 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 02:44:26 +0000 Subject: [PATCH 06/27] Update docker/setup-buildx-action action to v3 --- .github/workflows/build_and_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_publish.yml b/.github/workflows/build_and_publish.yml index 2aaf887..bd6478d 100644 --- a/.github/workflows/build_and_publish.yml +++ b/.github/workflows/build_and_publish.yml @@ -31,7 +31,7 @@ jobs: uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub if: github.event_name != 'pull_request' From 403912694258f5e6163d6ea95504eda45782fdc3 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 19 Dec 2023 15:02:40 +1300 Subject: [PATCH 07/27] Adds direct deletion --- internal/handler/main.go | 8 ++++++ internal/handler/messaging.go | 22 +++++++++++++++ internal/handler/processing.go | 28 +++++++++++++++++++ .../handler/testassets/deleteProblems.json | 6 ++++ internal/lagoonclient/problems.go | 1 + 5 files changed, 65 insertions(+) create mode 100644 internal/handler/testassets/deleteProblems.json diff --git a/internal/handler/main.go b/internal/handler/main.go index db7bf2a..bab2bcf 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -104,6 +104,14 @@ type DirectProblems struct { Problems []lagoonclient.LagoonProblem `json:"problems"` Type string `json:"type"` } + +type DirectDeleteMessage struct { + Type string `json:"type"` + EnvironmentId int `json:"environment"` + Source string `json:"source"` + Service string `json:"service"` +} + type InsightsData struct { InputType string InputPayload PayloadType diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index f8582cb..69d9ea0 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -88,6 +88,28 @@ func (h *Messaging) processMessageQueue(message mq.Message) { return } + // We also directly process deletion of problems and facts + if incoming.Type == "direct.delete.problems" { + slog.Debug("Deleting problems") + ret, err := deleteProblemsDirectly(message, h) + if err != nil { + slog.Error(err.Error()) + } + slog.Info(ret) + acknowledgeMessage() // Should we be acknowledging this error? + return + } + + if incoming.Type == "direct.delete.facts" { + ret, err := deleteFactsDirectly(message, h) + if err != nil { + + } + slog.Info(ret) + acknowledgeMessage() // Should we be acknowledging this error? + return + } + // Check labels for insights data from message if incoming.Labels != nil { labelKeys := make([]string, 0, len(incoming.Labels)) diff --git a/internal/handler/processing.go b/internal/handler/processing.go index a873fbb..51d6bcd 100644 --- a/internal/handler/processing.go +++ b/internal/handler/processing.go @@ -91,6 +91,34 @@ func processFactsDirectly(message mq.Message, h *Messaging) string { return facts } +func deleteProblemsDirectly(message mq.Message, h *Messaging) (string, error) { + var deleteMessage DirectDeleteMessage + err := json.Unmarshal(message.Body(), &deleteMessage) + if err != nil { + return "", err + } + ret, err := lagoonclient.DeleteProblemsFromSource(context.TODO(), h.getApiClient(), deleteMessage.EnvironmentId, deleteMessage.Service, deleteMessage.Source) + if err != nil { + slog.Error("Unable to delete facts", "Error", err.Error(), "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) + return "", err + } + return ret, nil +} + +func deleteFactsDirectly(message mq.Message, h *Messaging) (string, error) { + var deleteMessage DirectDeleteMessage + err := json.Unmarshal(message.Body(), &deleteMessage) + if err != nil { + return "", err + } + ret, err := lagoonclient.DeleteFactsFromSource(context.TODO(), h.getApiClient(), deleteMessage.EnvironmentId, deleteMessage.Source) + if err != nil { + slog.Error("Unable to delete facts", "Error", err.Error(), "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) + return "", err + } + return ret, nil +} + func processProblemsDirectly(message mq.Message, h *Messaging) ([]string, error) { var directProblems DirectProblems json.Unmarshal(message.Body(), &directProblems) diff --git a/internal/handler/testassets/deleteProblems.json b/internal/handler/testassets/deleteProblems.json new file mode 100644 index 0000000..3b521ea --- /dev/null +++ b/internal/handler/testassets/deleteProblems.json @@ -0,0 +1,6 @@ +{ + "environment": 3, + "type": "direct.delete.problems", + "source": "insights:facts:cli", + "service": "" +} diff --git a/internal/lagoonclient/problems.go b/internal/lagoonclient/problems.go index 56fe84f..a879ec7 100644 --- a/internal/lagoonclient/problems.go +++ b/internal/lagoonclient/problems.go @@ -59,6 +59,7 @@ func AddProblems(ctx context.Context, client graphql.Client, problems []LagoonPr } func DeleteProblemsFromSource(ctx context.Context, client graphql.Client, environmentID int, service string, source string) (string, error) { + resp, err := deleteProblemsFromSource(ctx, client, environmentID, source, service) if err != nil { return "", err From 2df7a816bf75554f7c3d9521a0bdf0583b561dea Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 16 Jan 2024 13:13:37 +1300 Subject: [PATCH 08/27] Further error processing --- internal/handler/insightsFactsParserFilter.go | 20 ++++++++++++------- internal/handler/insightsParserFilter.go | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/handler/insightsFactsParserFilter.go b/internal/handler/insightsFactsParserFilter.go index 885e07e..b2946a1 100644 --- a/internal/handler/insightsFactsParserFilter.go +++ b/internal/handler/insightsFactsParserFilter.go @@ -28,7 +28,11 @@ func processFactsInsightsData(h *Messaging, insights InsightsData, v string, api slog.Error("Error reading insights data", "Error", err) } - facts := processFactsFromJSON(logger, res, source) + facts, err := processFactsFromJSON(logger, res, source) + if err != nil { + return nil, "", err + } + facts, err = KeyFactsFilter(facts) if err != nil { return nil, "", err @@ -46,18 +50,17 @@ func processFactsInsightsData(h *Messaging, insights InsightsData, v string, api return nil, "", nil } -func processFactsFromJSON(logger *slog.Logger, facts []byte, source string) []LagoonFact { +func processFactsFromJSON(logger *slog.Logger, facts []byte, source string) ([]LagoonFact, error) { var factsInput []LagoonFact var factsPayload FactsPayload err := json.Unmarshal(facts, &factsPayload) if err != nil { - logger.Error(err.Error()) - panic("Can't unmarshal facts") + return factsInput, err } if len(factsPayload.Facts) == 0 { - return factsInput + return factsInput, nil } var filteredFacts []LagoonFact @@ -82,10 +85,13 @@ func processFactsFromJSON(logger *slog.Logger, facts []byte, source string) []La Type: FactTypeText, } logger.Debug("Processing fact", "name", f.Name, "value", f.Value) - fact, _ = ProcessLagoonFactAgainstRegisteredFilters(fact, f) + fact, err = ProcessLagoonFactAgainstRegisteredFilters(fact, f) + if err != nil { + return factsInput, err + } factsInput = append(factsInput, fact) } - return factsInput + return factsInput, nil } func init() { diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index 70950f9..3991975 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -54,7 +54,7 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC decoder := cdx.NewBOMDecoder(bytes.NewReader(b), cdx.BOMFileFormatJSON) if err = decoder.Decode(bom); err != nil { - panic(err) + return nil, "", err } } From 0b6c6ebbc4da17a9b4ae5343d5b440d933483eff Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Wed, 17 Jan 2024 09:34:27 +1300 Subject: [PATCH 09/27] Better logging --- internal/handler/messaging.go | 8 +++----- internal/handler/processing.go | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index 69d9ea0..7b879f2 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -91,21 +91,19 @@ func (h *Messaging) processMessageQueue(message mq.Message) { // We also directly process deletion of problems and facts if incoming.Type == "direct.delete.problems" { slog.Debug("Deleting problems") - ret, err := deleteProblemsDirectly(message, h) + _, err := deleteProblemsDirectly(message, h) if err != nil { slog.Error(err.Error()) } - slog.Info(ret) acknowledgeMessage() // Should we be acknowledging this error? return } if incoming.Type == "direct.delete.facts" { - ret, err := deleteFactsDirectly(message, h) + _, err := deleteFactsDirectly(message, h) if err != nil { - + slog.Error(err.Error()) } - slog.Info(ret) acknowledgeMessage() // Should we be acknowledging this error? return } diff --git a/internal/handler/processing.go b/internal/handler/processing.go index 51d6bcd..deb8413 100644 --- a/internal/handler/processing.go +++ b/internal/handler/processing.go @@ -102,6 +102,9 @@ func deleteProblemsDirectly(message mq.Message, h *Messaging) (string, error) { slog.Error("Unable to delete facts", "Error", err.Error(), "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) return "", err } + + slog.Info("Deleted problems", "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) + return ret, nil } @@ -116,6 +119,8 @@ func deleteFactsDirectly(message mq.Message, h *Messaging) (string, error) { slog.Error("Unable to delete facts", "Error", err.Error(), "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) return "", err } + slog.Info("Deleted facts", "EnvironmentId", deleteMessage.EnvironmentId, "source", deleteMessage.Source, "service", deleteMessage.Service) + return ret, nil } From b502213544f41ff697c06082f49754960982d2ca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 19:52:34 +0000 Subject: [PATCH 10/27] Update alpine Docker tag to v3.19 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index afb8239..b4455a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ FROM aquasec/trivy:0.48.0 as trivy # Refer to https://github.com/GoogleContainerTools/distroless for more details #FROM gcr.io/distroless/static:nonroot -FROM alpine:3.18 +FROM alpine:3.19 COPY --from=trivy /usr/local/bin/trivy /usr/local/bin/trivy From 64381fae340f97941ac71cd5b54fcd0184709c3a Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 5 Feb 2024 13:34:12 +1300 Subject: [PATCH 11/27] Updates cyclone-dx --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 2 ++ internal/handler/insightsParserFilter.go | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index afb8239..753e6ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY main.go main.go RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o insights-handler main.go # we pull the trivy binary from aquasec's alpine based image -FROM aquasec/trivy:0.48.0 as trivy +FROM aquasec/trivy:0.49.0 as trivy # Use distroless as minimal base image to package the insights-handler binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/go.mod b/go.mod index 0f10548..780cd6f 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ replace ( ) require ( - github.com/CycloneDX/cyclonedx-go v0.7.2-0.20230625092137-07e2f29defc3 + github.com/CycloneDX/cyclonedx-go v0.8.0 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect github.com/fsouza/go-dockerclient v1.7.3 // indirect github.com/minio/minio-go/v7 v7.0.21 diff --git a/go.sum b/go.sum index 10686dd..5b3412a 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/CycloneDX/cyclonedx-go v0.7.2-0.20230625092137-07e2f29defc3 h1:NqeV+ZMqpcosu0Xg2VW14Ru9ayBs/toe2oihS7sN6Xo= github.com/CycloneDX/cyclonedx-go v0.7.2-0.20230625092137-07e2f29defc3/go.mod h1:fGXSp1lCDfMQ8KR1EjxT4ewc5HHhGczRF2pWhLSWohs= +github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= +github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index 3991975..5db42a7 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -96,7 +96,7 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC //log.Printf("Successfully decoded SBOM of image %s with %s, found %d for '%s:%s'", bom.Metadata.Component.Name, (*bom.Metadata.Tools)[0].Name, len(*bom.Components), resource.Project, resource.Environment) logger.Info("Successfully decoded SBOM", "image", bom.Metadata.Component.Name, - "fieldName", (*bom.Metadata.Tools)[0].Name, + "fieldName", (*bom.Metadata.Tools.Components)[0].Name, "Length", len(*bom.Components), ) From 56ae401417f4ba202948520d574adbfce6f1b11d Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Wed, 7 Feb 2024 09:36:32 +1300 Subject: [PATCH 12/27] Adds imageInspectParserFilter_test --- .../handler/imageInspectParserFilter_test.go | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 internal/handler/imageInspectParserFilter_test.go diff --git a/internal/handler/imageInspectParserFilter_test.go b/internal/handler/imageInspectParserFilter_test.go new file mode 100644 index 0000000..52e6747 --- /dev/null +++ b/internal/handler/imageInspectParserFilter_test.go @@ -0,0 +1,79 @@ +package handler + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "testing" +) + +func Test_processFactsFromImageInspect(t *testing.T) { + type args struct { + logger *slog.Logger + imageInspectDataSource string + id int + source string + } + tests := []struct { + name string + args args + contains []LagoonFact + wantErr bool + }{ + { + name: "Testing Environment Variables", + args: args{ + logger: slog.Default(), + imageInspectDataSource: "testassets/imageInspectParserFilter/test1_envtesting.json", + id: 0, + source: "service", + }, + contains: []LagoonFact{ + { + Name: "PYTHON_PIP_VERSION", + Value: "1", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + imageInspectBytes, err := os.ReadFile(tt.args.imageInspectDataSource) + if err != nil { + slog.Error("Failed opening test file") + panic(1) + } + + imageInspectData := ImageData{} + + err = json.Unmarshal(imageInspectBytes, &imageInspectData) + if err != nil { + slog.Error(err.Error()) + panic(1) + } + + got, err := processFactsFromImageInspect(tt.args.logger, imageInspectData, tt.args.id, tt.args.source) + if (err != nil) != tt.wantErr { + t.Errorf("processFactsFromImageInspect() error = %v, wantErr %v", err, tt.wantErr) + return + } + + fmt.Println(got) + + for _, v := range tt.contains { + found := false + for _, f := range got { + if v.Name == f.Name && v.Value == f.Value { + found = true + break + } + } + if !found { + t.Errorf("processFactsFromImageInspect() could not find target values in data loaded from file") + } + } + }) + } +} From 7851e41436710a50fa904cb236592c6a2d84da69 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 03:53:54 +0000 Subject: [PATCH 13/27] Update golang Docker tag to v1.22.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 753e6ff..08e380d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.21.4-alpine3.18 as builder +FROM golang:1.22.0-alpine3.18 as builder COPY . /go/src/github.com/uselagoon/lagoon/services/insights-handler/ WORKDIR /go/src/github.com/uselagoon/lagoon/services/insights-handler/ From 6a19a41b70b0ed1847b285029dd00ad932de8e5e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 18:52:20 +0000 Subject: [PATCH 14/27] Update aquasec/trivy Docker tag to v0.49.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 69a166e..5c81883 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY main.go main.go RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o insights-handler main.go # we pull the trivy binary from aquasec's alpine based image -FROM aquasec/trivy:0.49.0 as trivy +FROM aquasec/trivy:0.49.1 as trivy # Use distroless as minimal base image to package the insights-handler binary # Refer to https://github.com/GoogleContainerTools/distroless for more details From 2c7f67e9fc79a2f6105506ac4c66915a986cda0d Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Thu, 8 Feb 2024 07:53:57 +1300 Subject: [PATCH 15/27] More tests and reorg --- internal/handler/messaging.go | 159 ++++--- .../imageInspectParserFilter/README.md | 3 + .../rawenvinspect_TOBEDELETED.json | 426 ++++++++++++++++++ .../test1_envtesting.json | 102 +++++ main.go | 11 +- 5 files changed, 617 insertions(+), 84 deletions(-) create mode 100644 internal/handler/testassets/imageInspectParserFilter/README.md create mode 100644 internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json create mode 100644 internal/handler/testassets/imageInspectParserFilter/test1_envtesting.json diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index aa760e2..8e8a3d6 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -37,8 +37,7 @@ func NewMessaging(config mq.Config, lagoonAPI LagoonAPI, s3 S3, startupAttempts // processMessageQueue reads in a rabbitMQ item and dispatches it to the appropriate function to process func (h *Messaging) processMessageQueue(message mq.Message) { - var insights InsightsData - var resource ResourceDestination + acknowledgeMessage := func(message mq.Message) func() { return func() { // Ack to remove from queue @@ -59,6 +58,9 @@ func (h *Messaging) processMessageQueue(message mq.Message) { } }(message) + // here we unmarshal the initial incoming message body + // notice how there is a "type" associated with the detail, + // this is the primary driver used to determine which subsystem this message will be processed by. incoming := &InsightsMessage{} err := json.Unmarshal(message.Body(), incoming) @@ -68,16 +70,13 @@ func (h *Messaging) processMessageQueue(message mq.Message) { return } - // if we have direct problems or facts, we process them differently - skipping all - // the extra processing below. - if incoming.Type == "direct.facts" { + switch incoming.Type { + case "direct.facts": resp := processFactsDirectly(message, h) slog.Debug(resp) acknowledgeMessage() return - } - - if incoming.Type == "direct.problems" { + case "direct.problems": resp, _ := processProblemsDirectly(message, h) if h.EnableDebug { for _, d := range resp { @@ -86,10 +85,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) { } acknowledgeMessage() return - } - - // We also directly process deletion of problems and facts - if incoming.Type == "direct.delete.problems" { + case "direct.delete.problems": slog.Debug("Deleting problems") _, err := deleteProblemsDirectly(message, h) if err != nil { @@ -97,9 +93,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) { } acknowledgeMessage() // Should we be acknowledging this error? return - } - - if incoming.Type == "direct.delete.facts" { + case "direct.delete.facts": _, err := deleteFactsDirectly(message, h) if err != nil { slog.Error(err.Error()) @@ -108,6 +102,62 @@ func (h *Messaging) processMessageQueue(message mq.Message) { return } + // If we get here, we don't have an assigned type - which means we process the data via inferrence. + // there are essentially two steps that happen there + // First - we preprocess and clean up the incoming data + // resource = contains details about where this came from + // insights = contains details about the actual insights data itself + resource, insights, err := preprocessIncomingMessageData(incoming) + + if err != nil { + slog.Error("Error preprocessing - rejecting message and exiting", "Error", err.Error()) + rejectMessage(false) + } + + slog.Debug("Insights", "data", fmt.Sprint(insights)) + slog.Debug("Target", "data", fmt.Sprint(resource)) + + // Process s3 upload + if !h.S3Config.Disabled { + if insights.InsightsType != Direct { + err := h.sendToLagoonS3(incoming, insights, resource) + if err != nil { + slog.Error("Unable to send to S3", "Error", err.Error()) + } + } + } + + // Process Lagoon API integration + if !h.LagoonAPI.Disabled { + if insights.InsightsType != Sbom && + insights.InsightsType != Image && + insights.InsightsType != Raw && + insights.InsightsType != Direct { + slog.Error("only 'sbom', 'direct', 'raw', and 'image' types are currently supported for api processing") + } else { + err := h.sendToLagoonAPI(incoming, resource, insights) + + if err != nil { + slog.Error("Unable to send to the API", "Error", err.Error()) + rejectMessage(false) + return + } + } + } + acknowledgeMessage() +} + +// preprocessIncomingMessageData deals with what are now legacy types, where most of the insight information +// used for further downstream processing is extracted from the message. +func preprocessIncomingMessageData(incoming *InsightsMessage) (ResourceDestination, InsightsData, error) { + var resource ResourceDestination + // Set some insight data defaults + insights := InsightsData{ + LagoonType: Facts, + OutputFileExt: "json", + OutputFileMIMEType: "application/json", + } + // Check labels for insights data from message if incoming.Labels != nil { labelKeys := make([]string, 0, len(incoming.Labels)) @@ -116,38 +166,25 @@ func (h *Messaging) processMessageQueue(message mq.Message) { } sort.Strings(labelKeys) - // Set some insight data defaults - insights = InsightsData{ - LagoonType: Facts, - OutputFileExt: "json", - OutputFileMIMEType: "application/json", - } - for _, label := range labelKeys { - if label == "lagoon.sh/project" { + switch label { + case "lagoon.sh/project": resource.Project = incoming.Labels["lagoon.sh/project"] - } - if label == "lagoon.sh/environment" { + case "lagoon.sh/environment": resource.Environment = incoming.Labels["lagoon.sh/environment"] - } - if label == "lagoon.sh/service" { + case "lagoon.sh/service": resource.Service = incoming.Labels["lagoon.sh/service"] - } - - if label == "lagoon.sh/insightsType" { + case "lagoon.sh/insightsType": insights.InputType = incoming.Labels["lagoon.sh/insightsType"] - } - if incoming.Labels["lagoon.sh/insightsType"] == "image-gz" { - insights.LagoonType = ImageFacts - } - if label == "lagoon.sh/insightsOutputCompressed" { + if incoming.Labels["lagoon.sh/insightsType"] == "image-gz" { + insights.LagoonType = ImageFacts + } + case "lagoon.sh/insightsOutputCompressed": compressed, _ := strconv.ParseBool(incoming.Labels["lagoon.sh/insightsOutputCompressed"]) insights.OutputCompressed = compressed - } - if label == "lagoon.sh/insightsOutputFileMIMEType" { + case "lagoon.sh/insightsOutputFileMIMEType": insights.OutputFileMIMEType = incoming.Labels["lagoon.sh/insightsOutputFileMIMEType"] - } - if label == "lagoon.sh/insightsOutputFileExt" { + case "lagoon.sh/insightsOutputFileExt": insights.OutputFileExt = incoming.Labels["lagoon.sh/insightsOutputFileExt"] } } @@ -169,9 +206,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) { // Determine incoming payload type if incoming.Payload == nil && incoming.BinaryPayload == nil { - slog.Debug("No payload was found - rejecting message and exiting") - rejectMessage(false) - return + return resource, insights, fmt.Errorf("No payload was found") } if len(incoming.Payload) != 0 { insights.InputPayload = Payload @@ -180,43 +215,5 @@ func (h *Messaging) processMessageQueue(message mq.Message) { insights.InputPayload = BinaryPayload } - // Debug - //if h.EnableDebug { - // log.Println("[DEBUG] insights:", insights) - // log.Println("[DEBUG] target:", resource) - //} - slog.Debug("Insights", "data", fmt.Sprint(insights)) - slog.Debug("Target", "data", fmt.Sprint(resource)) - - // Process s3 upload - if !h.S3Config.Disabled { - if insights.InsightsType != Direct { - err := h.sendToLagoonS3(incoming, insights, resource) - if err != nil { - //log.Printf("Unable to send to S3: %s", err.Error()) - slog.Error("Unable to send to S3", "Error", err.Error()) - - // TODO: BETTER ERROR HANDLING - } - } - } - - // Process Lagoon API integration - if !h.LagoonAPI.Disabled { - if insights.InsightsType != Sbom && - insights.InsightsType != Image && - insights.InsightsType != Raw && - insights.InsightsType != Direct { - slog.Error("only 'sbom', 'direct', 'raw', and 'image' types are currently supported for api processing") - } else { - err := h.sendToLagoonAPI(incoming, resource, insights) - - if err != nil { - slog.Error("Unable to send to the API", "Error", err.Error()) - rejectMessage(false) - return - } - } - } - acknowledgeMessage() + return resource, insights, nil } diff --git a/internal/handler/testassets/imageInspectParserFilter/README.md b/internal/handler/testassets/imageInspectParserFilter/README.md new file mode 100644 index 0000000..81d139b --- /dev/null +++ b/internal/handler/testassets/imageInspectParserFilter/README.md @@ -0,0 +1,3 @@ +# Asset description + +* test1_envtesting.json - contains a fully stacked "env" section that is used to ensure that parsing and checking environments works. \ No newline at end of file diff --git a/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json b/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json new file mode 100644 index 0000000..b968202 --- /dev/null +++ b/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json @@ -0,0 +1,426 @@ +{ + "Architecture": "amd64", + "Created": "2024-02-05T00:56:45.054544377Z", + "Digest": "sha256:6cb5dbcf5cc937d002152edcb92c0d19afd3407038ab0738a96a523879ba5e66", + "DockerVersion": "20.10.24", + "Env": [ + "PATH=/home/.composer/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "PHPIZE_DEPS=autoconf \t\tdpkg-dev dpkg \t\tfile \t\tg++ \t\tgcc \t\tlibc-dev \t\tmake \t\tpkgconf \t\tre2c", + "PHP_INI_DIR=/usr/local/etc/php", + "PHP_CFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", + "PHP_CPPFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", + "PHP_LDFLAGS=-Wl,-O1 -pie", + "PHP_VERSION=8.1.27", + "PHP_URL=https://www.php.net/distributions/php-8.1.27.tar.xz", + "PHP_ASC_URL=https://www.php.net/distributions/php-8.1.27.tar.xz.asc", + "PHP_SHA256=479e65c3f05714d4aace1370e617d78e49e996ec7a7579a5be47535be61f0658", + "LAGOON=cli-drupal", + "LAGOON_VERSION=24.1.0", + "TMPDIR=/tmp", + "TMP=/tmp", + "HOME=/home", + "ENV=/home/.bashrc", + "BASH_ENV=/home/.bashrc", + "NEWRELIC_VERSION=10.16.0.5", + "BLACKFIRE_VERSION=2.24.4", + "LAGOON_ENVIRONMENT_TYPE=development", + "COMPOSER_ALLOW_SUPERUSER=1", + "SSH_AUTH_SOCK=/tmp/ssh-agent", + "DRUSH_LAUNCHER_FALLBACK=/opt/drush8/vendor/bin/drush", + "WEBROOT=web", + "PYTHON_PIP_VERSION=1" + ], + "Labels": { + "maintainer": "The Lagoon Authors", + "org.opencontainers.image.authors": "The Lagoon Authors", + "org.opencontainers.image.source": "https://github.com/uselagoon/lagoon-images", + "repository": "https://github.com/uselagoon/lagoon-images" + }, + "Layers": [ + "sha256:c926b61bad3b94ae7351bafd0c184c159ebf0643b085f7ef1d47ecdc7316833c", + "sha256:bcd7d9d0a8568ad67e56f7560786ec06ada3d1624401e0c09a19bac68d6852bf", + "sha256:2ba0fbf849ad16a6e44bd1d2f64aad25339ce479d893afe5a2e6ab1d5f32a8b8", + "sha256:a37536836bf3207c08cde055f9b82489c03cdba8bf0195bcc3776a34fa0c4fc3", + "sha256:34dd1eb38c6eef42dc44ef78f8037758c11edf79e6f883b226f8368925679bf1", + "sha256:02578bf8264a5bbd28c725da9b14f535d03fbdc5f9c312c7ba7bb74012f4853d", + "sha256:13046608555fd11eae9875043459129a88d287c3968e94b7f91259fb81fd75a1", + "sha256:eb3d23206d3cb3bcde7b8b55a1e6fbb03c28dfd62d1bb3dee74d0d9935049dbe", + "sha256:0b7eabc24358763d4443a73b69df0dbf8253d95d298c23d51f5ef4ee8b86565a", + "sha256:bfc105895c2d186bc873dc5b92f46bc5fa5f8c8767d18891c8e16d684631edcd", + "sha256:9cf3e0c358986655cb43efebd042c6f169878f12a635309a4f74192e8ec9314f", + "sha256:2f0905b1e069960581752cbb042634deff15003eaa045107bf97f6d478f39552", + "sha256:2b337e0e352714b92696affda4f1315517fd2c86b9a3ebf881cb54a88d2e44b0", + "sha256:061dab159f320de642e68af2c6a6f43b2b2481f909a23a65ffa511056ebaa5ff", + "sha256:7d1bd404b2195cc1998fb5af7871e4e3b79baa0ce266a5a97ce63309eca4f7e0", + "sha256:40e1836cae5d5fb14f35435dab1472ae9acc7426b74fbbc76ebe6fe70d4ef3d7", + "sha256:fea74ff1b3da3d22d96ffea168a790aa06dcc83c3e38b7ceecd719a71f43bd53", + "sha256:81827ccbda885cef48a83b18dbda5fc1bf1758a8fb4fc986256e22f0390445a0", + "sha256:59d9f3e5da36ed4598144a5359d160030381364b50ebf74e9d2129db7cbf90cb", + "sha256:177b2f0ef828006dc545aa68c9c994eabdfea0533c28deedecdadf12d43d69ac", + "sha256:4ffe243db6afd8c85208ec65b8a7817330777926986f61d0d4cdc76024143064", + "sha256:208c1db53262c9d4051d96c121c513873eadabfa445c4b0b877b93fbdbb84b98", + "sha256:b8f29cedfe9c6260272d1b64116216e2704b4c8514cf5eb64797d6a64632de90", + "sha256:2c08cbaad66e4f860279d340c03962a57ae23142630c0efa95ee3f2367b9b43c", + "sha256:306d397f77e5336fe39a1a16a6825aeec41993adaa496b6079b02f1daa9d0ab6", + "sha256:9c440334b229f7c6e539fccee663e0ea118e68c283e26c9a1e708e8612f78635", + "sha256:b5a01e7c0bd466f128eba1533e51fb72b2fa7f83e9cee599db830f0ba808ea83", + "sha256:03cf1814b6168e25e8f30b32dda61bce99177b75636d9cafd6dc31196b7a60ab", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + "sha256:7d6f30c92bbc5505ccdf539f8a0ce2c2999657bf02a461ad84ad07083054e510", + "sha256:799bb018c57511bb6966c3b77095c9df063385a3eed3d594c8968f3843d84498", + "sha256:b6f3909bfe0b812b878e5415d70efd949d084700ac6bd47b8374b324433fcace", + "sha256:3feb76952714283cf6cdd7b9de88cab887c28c6f3603ec253949e4b6d3637edb", + "sha256:e4cf25d9bf1607e44f75bb81c6d8f31a8196e457e004f3685407574a50b5acaf", + "sha256:5aca279263ffe8ecde12d090947a62ed7305b71cbb46aadc5889e22d4cca48fb", + "sha256:0f2f54f263db81eb4837f648d747273f3fdcae63d1fb0ac1fa6b628f9c7e9837", + "sha256:e604343608e3bf1f1aeb1d559a0a6f6c94d38b582e4094d57ad1f4116d5f9200", + "sha256:49c6a3bffba4db85f0c178739267de92967fd1cbe66351f0ad71b0794a54f1dd", + "sha256:5323de1a26cb18ab72836a586f43f7ace71d4c3ae1d32bbdb605dbca24a1b2d5", + "sha256:110dfbac5781536633c8f4db803ac2d312635c138b6271950180d98edb25ad92", + "sha256:6459549ab2f0a5cbcd9459b7c0dd9d84d781e6a54de176bf7577eba0ef8fa717", + "sha256:064a69e5aa97e71c2a051b299f884d0d7cc5eb3a9874f701715196c689b6b0d9", + "sha256:ab9faf067017641a6a1a0cd5ca0a7822dce53cdcea43cc99935fa5782a504382", + "sha256:ab8cea428fbebc365aa1f17ebb2cd216c71125d12451f07ed08a20042bb95f2a", + "sha256:3b5c004c51ee98510d7d0db263d915bb7315ee5f24d63ebdc6f67c704e3e2aa6", + "sha256:82637563e123b17d0367abf543bdb5c464e3fe9be441a562c5b740998f6eed54", + "sha256:d1b66950bec6912c850243f1d5b32e80dffc34df5885f7586ae7cf69d3aba78a", + "sha256:556248b39507bff8be8a1d0b871d1460ef1125c270f671310aec774f90cc7fbb", + "sha256:619208f024fb9939ea6a8a935a539f985880e9cec47c7684a5850530c965d7fd", + "sha256:eb082d593603d8b87ed85cbfe5ce125325f46ac97fc2c468c4ae8fc6b46a5c5e", + "sha256:c385fbe872dfaf4fb44c7cbd9e67b093ef1eb5abae5ae4693c21808b8533043e", + "sha256:dd66dc57205d4b79bccdfaf5c0663dbe7e285265a43d6bd21116ff996abfd7c8", + "sha256:6360eea5a3c80db3710e24f12acd00c8bc642d9201910b8eb89a9729a2a5760f", + "sha256:2b62dc547d56dba81099f4713a8f35b3532d16f527e5068dadc7c7a0d06b7fc2" + ], + "LayersData": [ + { + "Annotations": null, + "Digest": "sha256:c926b61bad3b94ae7351bafd0c184c159ebf0643b085f7ef1d47ecdc7316833c", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 3402422 + }, + { + "Annotations": null, + "Digest": "sha256:bcd7d9d0a8568ad67e56f7560786ec06ada3d1624401e0c09a19bac68d6852bf", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 2682507 + }, + { + "Annotations": null, + "Digest": "sha256:2ba0fbf849ad16a6e44bd1d2f64aad25339ce479d893afe5a2e6ab1d5f32a8b8", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 1265 + }, + { + "Annotations": null, + "Digest": "sha256:a37536836bf3207c08cde055f9b82489c03cdba8bf0195bcc3776a34fa0c4fc3", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 268 + }, + { + "Annotations": null, + "Digest": "sha256:34dd1eb38c6eef42dc44ef78f8037758c11edf79e6f883b226f8368925679bf1", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 11935700 + }, + { + "Annotations": null, + "Digest": "sha256:02578bf8264a5bbd28c725da9b14f535d03fbdc5f9c312c7ba7bb74012f4853d", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 497 + }, + { + "Annotations": null, + "Digest": "sha256:13046608555fd11eae9875043459129a88d287c3968e94b7f91259fb81fd75a1", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 12428773 + }, + { + "Annotations": null, + "Digest": "sha256:eb3d23206d3cb3bcde7b8b55a1e6fbb03c28dfd62d1bb3dee74d0d9935049dbe", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 2450 + }, + { + "Annotations": null, + "Digest": "sha256:0b7eabc24358763d4443a73b69df0dbf8253d95d298c23d51f5ef4ee8b86565a", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 18954 + }, + { + "Annotations": null, + "Digest": "sha256:bfc105895c2d186bc873dc5b92f46bc5fa5f8c8767d18891c8e16d684631edcd", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 8879 + }, + { + "Annotations": null, + "Digest": "sha256:9cf3e0c358986655cb43efebd042c6f169878f12a635309a4f74192e8ec9314f", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 3594467 + }, + { + "Annotations": null, + "Digest": "sha256:2f0905b1e069960581752cbb042634deff15003eaa045107bf97f6d478f39552", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 2781913 + }, + { + "Annotations": null, + "Digest": "sha256:2b337e0e352714b92696affda4f1315517fd2c86b9a3ebf881cb54a88d2e44b0", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 9139 + }, + { + "Annotations": null, + "Digest": "sha256:061dab159f320de642e68af2c6a6f43b2b2481f909a23a65ffa511056ebaa5ff", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 631 + }, + { + "Annotations": null, + "Digest": "sha256:7d1bd404b2195cc1998fb5af7871e4e3b79baa0ce266a5a97ce63309eca4f7e0", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 524468 + }, + { + "Annotations": null, + "Digest": "sha256:40e1836cae5d5fb14f35435dab1472ae9acc7426b74fbbc76ebe6fe70d4ef3d7", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 610 + }, + { + "Annotations": null, + "Digest": "sha256:fea74ff1b3da3d22d96ffea168a790aa06dcc83c3e38b7ceecd719a71f43bd53", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 412 + }, + { + "Annotations": null, + "Digest": "sha256:81827ccbda885cef48a83b18dbda5fc1bf1758a8fb4fc986256e22f0390445a0", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 2350 + }, + { + "Annotations": null, + "Digest": "sha256:59d9f3e5da36ed4598144a5359d160030381364b50ebf74e9d2129db7cbf90cb", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 24074 + }, + { + "Annotations": null, + "Digest": "sha256:177b2f0ef828006dc545aa68c9c994eabdfea0533c28deedecdadf12d43d69ac", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 900 + }, + { + "Annotations": null, + "Digest": "sha256:4ffe243db6afd8c85208ec65b8a7817330777926986f61d0d4cdc76024143064", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 2356 + }, + { + "Annotations": null, + "Digest": "sha256:208c1db53262c9d4051d96c121c513873eadabfa445c4b0b877b93fbdbb84b98", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 264 + }, + { + "Annotations": null, + "Digest": "sha256:b8f29cedfe9c6260272d1b64116216e2704b4c8514cf5eb64797d6a64632de90", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 568 + }, + { + "Annotations": null, + "Digest": "sha256:2c08cbaad66e4f860279d340c03962a57ae23142630c0efa95ee3f2367b9b43c", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 56710179 + }, + { + "Annotations": null, + "Digest": "sha256:306d397f77e5336fe39a1a16a6825aeec41993adaa496b6079b02f1daa9d0ab6", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 9566830 + }, + { + "Annotations": null, + "Digest": "sha256:9c440334b229f7c6e539fccee663e0ea118e68c283e26c9a1e708e8612f78635", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 382605 + }, + { + "Annotations": null, + "Digest": "sha256:b5a01e7c0bd466f128eba1533e51fb72b2fa7f83e9cee599db830f0ba808ea83", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 11716541 + }, + { + "Annotations": null, + "Digest": "sha256:03cf1814b6168e25e8f30b32dda61bce99177b75636d9cafd6dc31196b7a60ab", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 93 + }, + { + "Annotations": null, + "Digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 32 + }, + { + "Annotations": null, + "Digest": "sha256:7d6f30c92bbc5505ccdf539f8a0ce2c2999657bf02a461ad84ad07083054e510", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 81715375 + }, + { + "Annotations": null, + "Digest": "sha256:799bb018c57511bb6966c3b77095c9df063385a3eed3d594c8968f3843d84498", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 705295 + }, + { + "Annotations": null, + "Digest": "sha256:b6f3909bfe0b812b878e5415d70efd949d084700ac6bd47b8374b324433fcace", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 1587 + }, + { + "Annotations": null, + "Digest": "sha256:3feb76952714283cf6cdd7b9de88cab887c28c6f3603ec253949e4b6d3637edb", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 616 + }, + { + "Annotations": null, + "Digest": "sha256:e4cf25d9bf1607e44f75bb81c6d8f31a8196e457e004f3685407574a50b5acaf", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 632 + }, + { + "Annotations": null, + "Digest": "sha256:5aca279263ffe8ecde12d090947a62ed7305b71cbb46aadc5889e22d4cca48fb", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 650 + }, + { + "Annotations": null, + "Digest": "sha256:0f2f54f263db81eb4837f648d747273f3fdcae63d1fb0ac1fa6b628f9c7e9837", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 659 + }, + { + "Annotations": null, + "Digest": "sha256:e604343608e3bf1f1aeb1d559a0a6f6c94d38b582e4094d57ad1f4116d5f9200", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 235 + }, + { + "Annotations": null, + "Digest": "sha256:49c6a3bffba4db85f0c178739267de92967fd1cbe66351f0ad71b0794a54f1dd", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 672 + }, + { + "Annotations": null, + "Digest": "sha256:5323de1a26cb18ab72836a586f43f7ace71d4c3ae1d32bbdb605dbca24a1b2d5", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 290 + }, + { + "Annotations": null, + "Digest": "sha256:110dfbac5781536633c8f4db803ac2d312635c138b6271950180d98edb25ad92", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 422 + }, + { + "Annotations": null, + "Digest": "sha256:6459549ab2f0a5cbcd9459b7c0dd9d84d781e6a54de176bf7577eba0ef8fa717", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 423 + }, + { + "Annotations": null, + "Digest": "sha256:064a69e5aa97e71c2a051b299f884d0d7cc5eb3a9874f701715196c689b6b0d9", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 900732 + }, + { + "Annotations": null, + "Digest": "sha256:ab9faf067017641a6a1a0cd5ca0a7822dce53cdcea43cc99935fa5782a504382", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 1525922 + }, + { + "Annotations": null, + "Digest": "sha256:ab8cea428fbebc365aa1f17ebb2cd216c71125d12451f07ed08a20042bb95f2a", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 54659 + }, + { + "Annotations": null, + "Digest": "sha256:3b5c004c51ee98510d7d0db263d915bb7315ee5f24d63ebdc6f67c704e3e2aa6", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 958 + }, + { + "Annotations": null, + "Digest": "sha256:82637563e123b17d0367abf543bdb5c464e3fe9be441a562c5b740998f6eed54", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 117 + }, + { + "Annotations": null, + "Digest": "sha256:d1b66950bec6912c850243f1d5b32e80dffc34df5885f7586ae7cf69d3aba78a", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 214 + }, + { + "Annotations": null, + "Digest": "sha256:556248b39507bff8be8a1d0b871d1460ef1125c270f671310aec774f90cc7fbb", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 455 + }, + { + "Annotations": null, + "Digest": "sha256:619208f024fb9939ea6a8a935a539f985880e9cec47c7684a5850530c965d7fd", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 321 + }, + { + "Annotations": null, + "Digest": "sha256:eb082d593603d8b87ed85cbfe5ce125325f46ac97fc2c468c4ae8fc6b46a5c5e", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 42812 + }, + { + "Annotations": null, + "Digest": "sha256:c385fbe872dfaf4fb44c7cbd9e67b093ef1eb5abae5ae4693c21808b8533043e", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 1434 + }, + { + "Annotations": null, + "Digest": "sha256:dd66dc57205d4b79bccdfaf5c0663dbe7e285265a43d6bd21116ff996abfd7c8", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 52666978 + }, + { + "Annotations": null, + "Digest": "sha256:6360eea5a3c80db3710e24f12acd00c8bc642d9201910b8eb89a9729a2a5760f", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 6531985 + }, + { + "Annotations": null, + "Digest": "sha256:2b62dc547d56dba81099f4713a8f35b3532d16f527e5068dadc7c7a0d06b7fc2", + "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "Size": 183 + } + ], + "Name": "test.lagoon.sh/projectname/envname/service", + "Os": "linux", + "RepoTags": [ + "latest" + ] +} \ No newline at end of file diff --git a/internal/handler/testassets/imageInspectParserFilter/test1_envtesting.json b/internal/handler/testassets/imageInspectParserFilter/test1_envtesting.json new file mode 100644 index 0000000..d687701 --- /dev/null +++ b/internal/handler/testassets/imageInspectParserFilter/test1_envtesting.json @@ -0,0 +1,102 @@ +{ + "Architecture": "amd64", + "Created": "2024-02-05T00:56:45.054544377Z", + "Digest": "sha256:6cb5dbcf5cc937d002152edcb92c0d19afd3407038ab0738a96a523879ba5e66", + "DockerVersion": "20.10.24", + "Env": [ + "PATH=/home/.composer/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "PHPIZE_DEPS=autoconf \t\tdpkg-dev dpkg \t\tfile \t\tg++ \t\tgcc \t\tlibc-dev \t\tmake \t\tpkgconf \t\tre2c", + "PHP_INI_DIR=/usr/local/etc/php", + "PHP_CFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", + "PHP_CPPFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", + "PHP_LDFLAGS=-Wl,-O1 -pie", + "PHP_VERSION=8.1.27", + "PHP_URL=https://www.php.net/distributions/php-8.1.27.tar.xz", + "PHP_ASC_URL=https://www.php.net/distributions/php-8.1.27.tar.xz.asc", + "PHP_SHA256=479e65c3f05714d4aace1370e617d78e49e996ec7a7579a5be47535be61f0658", + "LAGOON=cli-drupal", + "LAGOON_VERSION=24.1.0", + "TMPDIR=/tmp", + "TMP=/tmp", + "HOME=/home", + "ENV=/home/.bashrc", + "BASH_ENV=/home/.bashrc", + "NEWRELIC_VERSION=10.16.0.5", + "BLACKFIRE_VERSION=2.24.4", + "LAGOON_ENVIRONMENT_TYPE=development", + "COMPOSER_ALLOW_SUPERUSER=1", + "SSH_AUTH_SOCK=/tmp/ssh-agent", + "DRUSH_LAUNCHER_FALLBACK=/opt/drush8/vendor/bin/drush", + "WEBROOT=web", + "PYTHON_PIP_VERSION=1" + ], + "Labels": { + "maintainer": "The Lagoon Authors", + "org.opencontainers.image.authors": "The Lagoon Authors", + "org.opencontainers.image.source": "https://github.com/uselagoon/lagoon-images", + "repository": "https://github.com/uselagoon/lagoon-images" + }, + "Layers": [ + "sha256:c926b61bad3b94ae7351bafd0c184c159ebf0643b085f7ef1d47ecdc7316833c", + "sha256:bcd7d9d0a8568ad67e56f7560786ec06ada3d1624401e0c09a19bac68d6852bf", + "sha256:2ba0fbf849ad16a6e44bd1d2f64aad25339ce479d893afe5a2e6ab1d5f32a8b8", + "sha256:a37536836bf3207c08cde055f9b82489c03cdba8bf0195bcc3776a34fa0c4fc3", + "sha256:34dd1eb38c6eef42dc44ef78f8037758c11edf79e6f883b226f8368925679bf1", + "sha256:02578bf8264a5bbd28c725da9b14f535d03fbdc5f9c312c7ba7bb74012f4853d", + "sha256:13046608555fd11eae9875043459129a88d287c3968e94b7f91259fb81fd75a1", + "sha256:eb3d23206d3cb3bcde7b8b55a1e6fbb03c28dfd62d1bb3dee74d0d9935049dbe", + "sha256:0b7eabc24358763d4443a73b69df0dbf8253d95d298c23d51f5ef4ee8b86565a", + "sha256:bfc105895c2d186bc873dc5b92f46bc5fa5f8c8767d18891c8e16d684631edcd", + "sha256:9cf3e0c358986655cb43efebd042c6f169878f12a635309a4f74192e8ec9314f", + "sha256:2f0905b1e069960581752cbb042634deff15003eaa045107bf97f6d478f39552", + "sha256:2b337e0e352714b92696affda4f1315517fd2c86b9a3ebf881cb54a88d2e44b0", + "sha256:061dab159f320de642e68af2c6a6f43b2b2481f909a23a65ffa511056ebaa5ff", + "sha256:7d1bd404b2195cc1998fb5af7871e4e3b79baa0ce266a5a97ce63309eca4f7e0", + "sha256:40e1836cae5d5fb14f35435dab1472ae9acc7426b74fbbc76ebe6fe70d4ef3d7", + "sha256:fea74ff1b3da3d22d96ffea168a790aa06dcc83c3e38b7ceecd719a71f43bd53", + "sha256:81827ccbda885cef48a83b18dbda5fc1bf1758a8fb4fc986256e22f0390445a0", + "sha256:59d9f3e5da36ed4598144a5359d160030381364b50ebf74e9d2129db7cbf90cb", + "sha256:177b2f0ef828006dc545aa68c9c994eabdfea0533c28deedecdadf12d43d69ac", + "sha256:4ffe243db6afd8c85208ec65b8a7817330777926986f61d0d4cdc76024143064", + "sha256:208c1db53262c9d4051d96c121c513873eadabfa445c4b0b877b93fbdbb84b98", + "sha256:b8f29cedfe9c6260272d1b64116216e2704b4c8514cf5eb64797d6a64632de90", + "sha256:2c08cbaad66e4f860279d340c03962a57ae23142630c0efa95ee3f2367b9b43c", + "sha256:306d397f77e5336fe39a1a16a6825aeec41993adaa496b6079b02f1daa9d0ab6", + "sha256:9c440334b229f7c6e539fccee663e0ea118e68c283e26c9a1e708e8612f78635", + "sha256:b5a01e7c0bd466f128eba1533e51fb72b2fa7f83e9cee599db830f0ba808ea83", + "sha256:03cf1814b6168e25e8f30b32dda61bce99177b75636d9cafd6dc31196b7a60ab", + "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", + "sha256:7d6f30c92bbc5505ccdf539f8a0ce2c2999657bf02a461ad84ad07083054e510", + "sha256:799bb018c57511bb6966c3b77095c9df063385a3eed3d594c8968f3843d84498", + "sha256:b6f3909bfe0b812b878e5415d70efd949d084700ac6bd47b8374b324433fcace", + "sha256:3feb76952714283cf6cdd7b9de88cab887c28c6f3603ec253949e4b6d3637edb", + "sha256:e4cf25d9bf1607e44f75bb81c6d8f31a8196e457e004f3685407574a50b5acaf", + "sha256:5aca279263ffe8ecde12d090947a62ed7305b71cbb46aadc5889e22d4cca48fb", + "sha256:0f2f54f263db81eb4837f648d747273f3fdcae63d1fb0ac1fa6b628f9c7e9837", + "sha256:e604343608e3bf1f1aeb1d559a0a6f6c94d38b582e4094d57ad1f4116d5f9200", + "sha256:49c6a3bffba4db85f0c178739267de92967fd1cbe66351f0ad71b0794a54f1dd", + "sha256:5323de1a26cb18ab72836a586f43f7ace71d4c3ae1d32bbdb605dbca24a1b2d5", + "sha256:110dfbac5781536633c8f4db803ac2d312635c138b6271950180d98edb25ad92", + "sha256:6459549ab2f0a5cbcd9459b7c0dd9d84d781e6a54de176bf7577eba0ef8fa717", + "sha256:064a69e5aa97e71c2a051b299f884d0d7cc5eb3a9874f701715196c689b6b0d9", + "sha256:ab9faf067017641a6a1a0cd5ca0a7822dce53cdcea43cc99935fa5782a504382", + "sha256:ab8cea428fbebc365aa1f17ebb2cd216c71125d12451f07ed08a20042bb95f2a", + "sha256:3b5c004c51ee98510d7d0db263d915bb7315ee5f24d63ebdc6f67c704e3e2aa6", + "sha256:82637563e123b17d0367abf543bdb5c464e3fe9be441a562c5b740998f6eed54", + "sha256:d1b66950bec6912c850243f1d5b32e80dffc34df5885f7586ae7cf69d3aba78a", + "sha256:556248b39507bff8be8a1d0b871d1460ef1125c270f671310aec774f90cc7fbb", + "sha256:619208f024fb9939ea6a8a935a539f985880e9cec47c7684a5850530c965d7fd", + "sha256:eb082d593603d8b87ed85cbfe5ce125325f46ac97fc2c468c4ae8fc6b46a5c5e", + "sha256:c385fbe872dfaf4fb44c7cbd9e67b093ef1eb5abae5ae4693c21808b8533043e", + "sha256:dd66dc57205d4b79bccdfaf5c0663dbe7e285265a43d6bd21116ff996abfd7c8", + "sha256:6360eea5a3c80db3710e24f12acd00c8bc642d9201910b8eb89a9729a2a5760f", + "sha256:2b62dc547d56dba81099f4713a8f35b3532d16f527e5068dadc7c7a0d06b7fc2" + ], + "LayersData": [ + ], + "Name": "test.lagoon.sh/projectname/envname/service", + "Os": "linux", + "RepoTags": [ + "latest" + ] +} \ No newline at end of file diff --git a/main.go b/main.go index 9c23bca..63f4771 100644 --- a/main.go +++ b/main.go @@ -122,7 +122,7 @@ func main() { } } - // configure the backup handler settings + // We begin by setting up the handler's broker connection broker := handler.RabbitBroker{ Hostname: fmt.Sprintf("%s:%s", mqHost, mqPort), Username: mqUser, @@ -130,6 +130,8 @@ func main() { QueueName: insightsQueueName, ExchangeName: insightsExchange, } + + // graphQLConfig details how we connect to the Lagoon API graphQLConfig := handler.LagoonAPI{ Endpoint: lagoonAPIHost, TokenSigningKey: jwtTokenSigningKey, @@ -138,6 +140,8 @@ func main() { JWTIssuer: jwtIssuer, Disabled: disableAPIIntegration, } + + // s3Config details how we connect to the s3 buckets - these are used to upload files s3Config := handler.S3{ SecretAccessKey: s3SecretAccessKey, S3Origin: s3Origin, @@ -150,10 +154,11 @@ func main() { slog.Debug("disableS3Upload", "status", disableS3Upload) + // Here we look at the filter json/yaml and attempt to load up the filter descriptions err := handler.RegisterFiltersFromDisk(filterTransformerFile) if err != nil { - // TODO: BETTER ERROR HANDLING slog.Error("Unable to register filters from disk", "Error", err) + os.Exit(1) } config := mq.Config{ @@ -209,7 +214,7 @@ func main() { ) // start the consumer - //slog.Info("insights-handler is started-up") + slog.Info("insights-handler has started-up") messaging.Consumer() } From 7765f6d13aa39ac26f3173ddd47feda76d40d9e6 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Thu, 8 Feb 2024 12:36:19 +1300 Subject: [PATCH 16/27] Refactoring to types --- internal/handler/main.go | 30 +++++++----------------------- internal/handler/parserfilter.go | 20 +++++--------------- 2 files changed, 12 insertions(+), 38 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index c99e78a..5ef386b 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -329,7 +329,7 @@ func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messa func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) error { for _, filter := range parserFilters { - var result []interface{} + var result []LagoonFact var source string json, err := json.Marshal(p) @@ -353,7 +353,7 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa } // processResultset will send results as facts to the lagoon api after processing via a parser filter -func processResultset(result []interface{}, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { +func processResultset(result []LagoonFact, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { project, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { slog.Error(apiErr.Error()) @@ -368,28 +368,12 @@ func processResultset(result []interface{}, err error, h *Messaging, apiClient g return apiErr } - for _, r := range result { - if fact, ok := r.(LagoonFact); ok { - // Handle single fact - err = h.sendFactsToLagoonAPI([]LagoonFact{fact}, apiClient, resource, source) - if err != nil { - slog.Error("Error sending facts to Lagoon API", "error", err.Error()) - return err - } - } else if facts, ok := r.([]LagoonFact); ok { - // Handle slice of facts - err = h.sendFactsToLagoonAPI(facts, apiClient, resource, source) - if err != nil { - slog.Error("Error sending facts to Lagoon API", "error", err.Error()) - return err - } - } else { - // Unexpected type returned from filter() - err := fmt.Errorf("unexpected type returned from filter(): %T\n", r) - slog.Error(err.Error()) - return err - } + err = h.sendFactsToLagoonAPI(result, apiClient, resource, source) + if err != nil { + slog.Error("Error sending facts to Lagoon API", "error", err.Error()) + return err } + return nil } diff --git a/internal/handler/parserfilter.go b/internal/handler/parserfilter.go index 5995f93..ab205c0 100644 --- a/internal/handler/parserfilter.go +++ b/internal/handler/parserfilter.go @@ -16,26 +16,16 @@ type parserFilter interface { getFact() LagoonFact } -type ParserFilterFunc[T any] func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]T, string, error) +type ParserFilterFunc func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) -var parserFilters []ParserFilterFunc[interface{}] +var parserFilters []ParserFilterFunc -// Since Go does not allow type conversions between slices of different types, ([]T and []interface{}) are considered different types, and you cannot assign one to the other. -// Therefore this func will convert the []T slice to a []interface{} slice -func ToInterfaceSlice[T any](slice []T) []interface{} { - result := make([]interface{}, len(slice)) - for i, v := range slice { - result[i] = v - } - return result -} - -func RegisterParserFilter[T any](pf ParserFilterFunc[T]) { - parserFilters = append(parserFilters, func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]interface{}, string, error) { +func RegisterParserFilter(pf ParserFilterFunc) { + parserFilters = append(parserFilters, func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) { result, source, err := pf(h, insights, v, apiClient, resource) if err != nil { return nil, "", err } - return ToInterfaceSlice(result), source, nil + return result, source, nil }) } From 478ce1936c16cd0f5b56197c910293d1fc027b87 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 9 Feb 2024 08:11:16 +1300 Subject: [PATCH 17/27] Refactor functionality of gathering facts from writing them --- internal/handler/main.go | 50 +- .../rawenvinspect_TOBEDELETED.json | 426 ------------------ 2 files changed, 29 insertions(+), 447 deletions(-) delete mode 100644 internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json diff --git a/internal/handler/main.go b/internal/handler/main.go index 5ef386b..d7528f8 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -281,6 +281,8 @@ func (t *authedTransport) RoundTrip(req *http.Request) (*http.Response, error) { return t.wrapped.RoundTrip(req) } +type LagoonSourceFactMap map[string][]LagoonFact + // Incoming payload may contain facts or problems, so we need to handle these differently func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource ResourceDestination, insights InsightsData) (err error) { apiClient := h.getApiClient() @@ -291,43 +293,53 @@ func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource Resource if insights.InputPayload == Payload && insights.LagoonType == Facts { for _, p := range incoming.Payload { - err := parserFilterLoopForPayloads(insights, p, h, apiClient, resource) + lagoonSourceFactMap, err := parserFilterLoopForPayloads(insights, p, h, apiClient, resource) if err != nil { return err } + for sourceName, facts := range lagoonSourceFactMap { + err = sendResultsetToLagoon(facts, err, h, apiClient, resource, sourceName) + if err != nil { + return err + } + } } } if insights.InputPayload == BinaryPayload && insights.LagoonType == Facts { for _, p := range incoming.BinaryPayload { - err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) + lagoonSourceFactMap, err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) if err != nil { return err } + for sourceName, facts := range lagoonSourceFactMap { + err = sendResultsetToLagoon(facts, err, h, apiClient, resource, sourceName) + if err != nil { + return err + } + } } } return nil } -func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) error { +func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) (LagoonSourceFactMap, error) { + lagoonSourceFactMap := LagoonSourceFactMap{} for _, filter := range parserFilters { result, source, err := filter(h, insights, p, apiClient, resource) if err != nil { slog.Error("Error running filter", "error", err.Error()) - return err - } - - err = processResultset(result, err, h, apiClient, resource, source) - if err != nil { - return err + return lagoonSourceFactMap, err } + lagoonSourceFactMap[source] = result } - return nil + return lagoonSourceFactMap, nil } -func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) error { +func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) (LagoonSourceFactMap, error) { + lagoonSourceFactMap := LagoonSourceFactMap{} for _, filter := range parserFilters { var result []LagoonFact var source string @@ -335,25 +347,21 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa json, err := json.Marshal(p) if err != nil { slog.Error("Error marshalling data", "error", err.Error()) - return err + return lagoonSourceFactMap, err } result, source, err = filter(h, insights, fmt.Sprintf("%s", json), apiClient, resource) if err != nil { slog.Error("Error Filtering payload", "error", err.Error()) - return err - } - - err = processResultset(result, err, h, apiClient, resource, source) - if err != nil { - return err + return lagoonSourceFactMap, err } + lagoonSourceFactMap[source] = result } - return nil + return lagoonSourceFactMap, nil } -// processResultset will send results as facts to the lagoon api after processing via a parser filter -func processResultset(result []LagoonFact, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { +// sendResultsetToLagoon will send results as facts to the lagoon api after processing via a parser filter +func sendResultsetToLagoon(result []LagoonFact, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { project, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { slog.Error(apiErr.Error()) diff --git a/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json b/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json deleted file mode 100644 index b968202..0000000 --- a/internal/handler/testassets/imageInspectParserFilter/rawenvinspect_TOBEDELETED.json +++ /dev/null @@ -1,426 +0,0 @@ -{ - "Architecture": "amd64", - "Created": "2024-02-05T00:56:45.054544377Z", - "Digest": "sha256:6cb5dbcf5cc937d002152edcb92c0d19afd3407038ab0738a96a523879ba5e66", - "DockerVersion": "20.10.24", - "Env": [ - "PATH=/home/.composer/vendor/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", - "PHPIZE_DEPS=autoconf \t\tdpkg-dev dpkg \t\tfile \t\tg++ \t\tgcc \t\tlibc-dev \t\tmake \t\tpkgconf \t\tre2c", - "PHP_INI_DIR=/usr/local/etc/php", - "PHP_CFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", - "PHP_CPPFLAGS=-fstack-protector-strong -fpic -fpie -O2 -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64", - "PHP_LDFLAGS=-Wl,-O1 -pie", - "PHP_VERSION=8.1.27", - "PHP_URL=https://www.php.net/distributions/php-8.1.27.tar.xz", - "PHP_ASC_URL=https://www.php.net/distributions/php-8.1.27.tar.xz.asc", - "PHP_SHA256=479e65c3f05714d4aace1370e617d78e49e996ec7a7579a5be47535be61f0658", - "LAGOON=cli-drupal", - "LAGOON_VERSION=24.1.0", - "TMPDIR=/tmp", - "TMP=/tmp", - "HOME=/home", - "ENV=/home/.bashrc", - "BASH_ENV=/home/.bashrc", - "NEWRELIC_VERSION=10.16.0.5", - "BLACKFIRE_VERSION=2.24.4", - "LAGOON_ENVIRONMENT_TYPE=development", - "COMPOSER_ALLOW_SUPERUSER=1", - "SSH_AUTH_SOCK=/tmp/ssh-agent", - "DRUSH_LAUNCHER_FALLBACK=/opt/drush8/vendor/bin/drush", - "WEBROOT=web", - "PYTHON_PIP_VERSION=1" - ], - "Labels": { - "maintainer": "The Lagoon Authors", - "org.opencontainers.image.authors": "The Lagoon Authors", - "org.opencontainers.image.source": "https://github.com/uselagoon/lagoon-images", - "repository": "https://github.com/uselagoon/lagoon-images" - }, - "Layers": [ - "sha256:c926b61bad3b94ae7351bafd0c184c159ebf0643b085f7ef1d47ecdc7316833c", - "sha256:bcd7d9d0a8568ad67e56f7560786ec06ada3d1624401e0c09a19bac68d6852bf", - "sha256:2ba0fbf849ad16a6e44bd1d2f64aad25339ce479d893afe5a2e6ab1d5f32a8b8", - "sha256:a37536836bf3207c08cde055f9b82489c03cdba8bf0195bcc3776a34fa0c4fc3", - "sha256:34dd1eb38c6eef42dc44ef78f8037758c11edf79e6f883b226f8368925679bf1", - "sha256:02578bf8264a5bbd28c725da9b14f535d03fbdc5f9c312c7ba7bb74012f4853d", - "sha256:13046608555fd11eae9875043459129a88d287c3968e94b7f91259fb81fd75a1", - "sha256:eb3d23206d3cb3bcde7b8b55a1e6fbb03c28dfd62d1bb3dee74d0d9935049dbe", - "sha256:0b7eabc24358763d4443a73b69df0dbf8253d95d298c23d51f5ef4ee8b86565a", - "sha256:bfc105895c2d186bc873dc5b92f46bc5fa5f8c8767d18891c8e16d684631edcd", - "sha256:9cf3e0c358986655cb43efebd042c6f169878f12a635309a4f74192e8ec9314f", - "sha256:2f0905b1e069960581752cbb042634deff15003eaa045107bf97f6d478f39552", - "sha256:2b337e0e352714b92696affda4f1315517fd2c86b9a3ebf881cb54a88d2e44b0", - "sha256:061dab159f320de642e68af2c6a6f43b2b2481f909a23a65ffa511056ebaa5ff", - "sha256:7d1bd404b2195cc1998fb5af7871e4e3b79baa0ce266a5a97ce63309eca4f7e0", - "sha256:40e1836cae5d5fb14f35435dab1472ae9acc7426b74fbbc76ebe6fe70d4ef3d7", - "sha256:fea74ff1b3da3d22d96ffea168a790aa06dcc83c3e38b7ceecd719a71f43bd53", - "sha256:81827ccbda885cef48a83b18dbda5fc1bf1758a8fb4fc986256e22f0390445a0", - "sha256:59d9f3e5da36ed4598144a5359d160030381364b50ebf74e9d2129db7cbf90cb", - "sha256:177b2f0ef828006dc545aa68c9c994eabdfea0533c28deedecdadf12d43d69ac", - "sha256:4ffe243db6afd8c85208ec65b8a7817330777926986f61d0d4cdc76024143064", - "sha256:208c1db53262c9d4051d96c121c513873eadabfa445c4b0b877b93fbdbb84b98", - "sha256:b8f29cedfe9c6260272d1b64116216e2704b4c8514cf5eb64797d6a64632de90", - "sha256:2c08cbaad66e4f860279d340c03962a57ae23142630c0efa95ee3f2367b9b43c", - "sha256:306d397f77e5336fe39a1a16a6825aeec41993adaa496b6079b02f1daa9d0ab6", - "sha256:9c440334b229f7c6e539fccee663e0ea118e68c283e26c9a1e708e8612f78635", - "sha256:b5a01e7c0bd466f128eba1533e51fb72b2fa7f83e9cee599db830f0ba808ea83", - "sha256:03cf1814b6168e25e8f30b32dda61bce99177b75636d9cafd6dc31196b7a60ab", - "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", - "sha256:7d6f30c92bbc5505ccdf539f8a0ce2c2999657bf02a461ad84ad07083054e510", - "sha256:799bb018c57511bb6966c3b77095c9df063385a3eed3d594c8968f3843d84498", - "sha256:b6f3909bfe0b812b878e5415d70efd949d084700ac6bd47b8374b324433fcace", - "sha256:3feb76952714283cf6cdd7b9de88cab887c28c6f3603ec253949e4b6d3637edb", - "sha256:e4cf25d9bf1607e44f75bb81c6d8f31a8196e457e004f3685407574a50b5acaf", - "sha256:5aca279263ffe8ecde12d090947a62ed7305b71cbb46aadc5889e22d4cca48fb", - "sha256:0f2f54f263db81eb4837f648d747273f3fdcae63d1fb0ac1fa6b628f9c7e9837", - "sha256:e604343608e3bf1f1aeb1d559a0a6f6c94d38b582e4094d57ad1f4116d5f9200", - "sha256:49c6a3bffba4db85f0c178739267de92967fd1cbe66351f0ad71b0794a54f1dd", - "sha256:5323de1a26cb18ab72836a586f43f7ace71d4c3ae1d32bbdb605dbca24a1b2d5", - "sha256:110dfbac5781536633c8f4db803ac2d312635c138b6271950180d98edb25ad92", - "sha256:6459549ab2f0a5cbcd9459b7c0dd9d84d781e6a54de176bf7577eba0ef8fa717", - "sha256:064a69e5aa97e71c2a051b299f884d0d7cc5eb3a9874f701715196c689b6b0d9", - "sha256:ab9faf067017641a6a1a0cd5ca0a7822dce53cdcea43cc99935fa5782a504382", - "sha256:ab8cea428fbebc365aa1f17ebb2cd216c71125d12451f07ed08a20042bb95f2a", - "sha256:3b5c004c51ee98510d7d0db263d915bb7315ee5f24d63ebdc6f67c704e3e2aa6", - "sha256:82637563e123b17d0367abf543bdb5c464e3fe9be441a562c5b740998f6eed54", - "sha256:d1b66950bec6912c850243f1d5b32e80dffc34df5885f7586ae7cf69d3aba78a", - "sha256:556248b39507bff8be8a1d0b871d1460ef1125c270f671310aec774f90cc7fbb", - "sha256:619208f024fb9939ea6a8a935a539f985880e9cec47c7684a5850530c965d7fd", - "sha256:eb082d593603d8b87ed85cbfe5ce125325f46ac97fc2c468c4ae8fc6b46a5c5e", - "sha256:c385fbe872dfaf4fb44c7cbd9e67b093ef1eb5abae5ae4693c21808b8533043e", - "sha256:dd66dc57205d4b79bccdfaf5c0663dbe7e285265a43d6bd21116ff996abfd7c8", - "sha256:6360eea5a3c80db3710e24f12acd00c8bc642d9201910b8eb89a9729a2a5760f", - "sha256:2b62dc547d56dba81099f4713a8f35b3532d16f527e5068dadc7c7a0d06b7fc2" - ], - "LayersData": [ - { - "Annotations": null, - "Digest": "sha256:c926b61bad3b94ae7351bafd0c184c159ebf0643b085f7ef1d47ecdc7316833c", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 3402422 - }, - { - "Annotations": null, - "Digest": "sha256:bcd7d9d0a8568ad67e56f7560786ec06ada3d1624401e0c09a19bac68d6852bf", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 2682507 - }, - { - "Annotations": null, - "Digest": "sha256:2ba0fbf849ad16a6e44bd1d2f64aad25339ce479d893afe5a2e6ab1d5f32a8b8", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 1265 - }, - { - "Annotations": null, - "Digest": "sha256:a37536836bf3207c08cde055f9b82489c03cdba8bf0195bcc3776a34fa0c4fc3", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 268 - }, - { - "Annotations": null, - "Digest": "sha256:34dd1eb38c6eef42dc44ef78f8037758c11edf79e6f883b226f8368925679bf1", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 11935700 - }, - { - "Annotations": null, - "Digest": "sha256:02578bf8264a5bbd28c725da9b14f535d03fbdc5f9c312c7ba7bb74012f4853d", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 497 - }, - { - "Annotations": null, - "Digest": "sha256:13046608555fd11eae9875043459129a88d287c3968e94b7f91259fb81fd75a1", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 12428773 - }, - { - "Annotations": null, - "Digest": "sha256:eb3d23206d3cb3bcde7b8b55a1e6fbb03c28dfd62d1bb3dee74d0d9935049dbe", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 2450 - }, - { - "Annotations": null, - "Digest": "sha256:0b7eabc24358763d4443a73b69df0dbf8253d95d298c23d51f5ef4ee8b86565a", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 18954 - }, - { - "Annotations": null, - "Digest": "sha256:bfc105895c2d186bc873dc5b92f46bc5fa5f8c8767d18891c8e16d684631edcd", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 8879 - }, - { - "Annotations": null, - "Digest": "sha256:9cf3e0c358986655cb43efebd042c6f169878f12a635309a4f74192e8ec9314f", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 3594467 - }, - { - "Annotations": null, - "Digest": "sha256:2f0905b1e069960581752cbb042634deff15003eaa045107bf97f6d478f39552", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 2781913 - }, - { - "Annotations": null, - "Digest": "sha256:2b337e0e352714b92696affda4f1315517fd2c86b9a3ebf881cb54a88d2e44b0", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 9139 - }, - { - "Annotations": null, - "Digest": "sha256:061dab159f320de642e68af2c6a6f43b2b2481f909a23a65ffa511056ebaa5ff", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 631 - }, - { - "Annotations": null, - "Digest": "sha256:7d1bd404b2195cc1998fb5af7871e4e3b79baa0ce266a5a97ce63309eca4f7e0", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 524468 - }, - { - "Annotations": null, - "Digest": "sha256:40e1836cae5d5fb14f35435dab1472ae9acc7426b74fbbc76ebe6fe70d4ef3d7", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 610 - }, - { - "Annotations": null, - "Digest": "sha256:fea74ff1b3da3d22d96ffea168a790aa06dcc83c3e38b7ceecd719a71f43bd53", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 412 - }, - { - "Annotations": null, - "Digest": "sha256:81827ccbda885cef48a83b18dbda5fc1bf1758a8fb4fc986256e22f0390445a0", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 2350 - }, - { - "Annotations": null, - "Digest": "sha256:59d9f3e5da36ed4598144a5359d160030381364b50ebf74e9d2129db7cbf90cb", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 24074 - }, - { - "Annotations": null, - "Digest": "sha256:177b2f0ef828006dc545aa68c9c994eabdfea0533c28deedecdadf12d43d69ac", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 900 - }, - { - "Annotations": null, - "Digest": "sha256:4ffe243db6afd8c85208ec65b8a7817330777926986f61d0d4cdc76024143064", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 2356 - }, - { - "Annotations": null, - "Digest": "sha256:208c1db53262c9d4051d96c121c513873eadabfa445c4b0b877b93fbdbb84b98", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 264 - }, - { - "Annotations": null, - "Digest": "sha256:b8f29cedfe9c6260272d1b64116216e2704b4c8514cf5eb64797d6a64632de90", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 568 - }, - { - "Annotations": null, - "Digest": "sha256:2c08cbaad66e4f860279d340c03962a57ae23142630c0efa95ee3f2367b9b43c", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 56710179 - }, - { - "Annotations": null, - "Digest": "sha256:306d397f77e5336fe39a1a16a6825aeec41993adaa496b6079b02f1daa9d0ab6", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 9566830 - }, - { - "Annotations": null, - "Digest": "sha256:9c440334b229f7c6e539fccee663e0ea118e68c283e26c9a1e708e8612f78635", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 382605 - }, - { - "Annotations": null, - "Digest": "sha256:b5a01e7c0bd466f128eba1533e51fb72b2fa7f83e9cee599db830f0ba808ea83", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 11716541 - }, - { - "Annotations": null, - "Digest": "sha256:03cf1814b6168e25e8f30b32dda61bce99177b75636d9cafd6dc31196b7a60ab", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 93 - }, - { - "Annotations": null, - "Digest": "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 32 - }, - { - "Annotations": null, - "Digest": "sha256:7d6f30c92bbc5505ccdf539f8a0ce2c2999657bf02a461ad84ad07083054e510", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 81715375 - }, - { - "Annotations": null, - "Digest": "sha256:799bb018c57511bb6966c3b77095c9df063385a3eed3d594c8968f3843d84498", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 705295 - }, - { - "Annotations": null, - "Digest": "sha256:b6f3909bfe0b812b878e5415d70efd949d084700ac6bd47b8374b324433fcace", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 1587 - }, - { - "Annotations": null, - "Digest": "sha256:3feb76952714283cf6cdd7b9de88cab887c28c6f3603ec253949e4b6d3637edb", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 616 - }, - { - "Annotations": null, - "Digest": "sha256:e4cf25d9bf1607e44f75bb81c6d8f31a8196e457e004f3685407574a50b5acaf", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 632 - }, - { - "Annotations": null, - "Digest": "sha256:5aca279263ffe8ecde12d090947a62ed7305b71cbb46aadc5889e22d4cca48fb", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 650 - }, - { - "Annotations": null, - "Digest": "sha256:0f2f54f263db81eb4837f648d747273f3fdcae63d1fb0ac1fa6b628f9c7e9837", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 659 - }, - { - "Annotations": null, - "Digest": "sha256:e604343608e3bf1f1aeb1d559a0a6f6c94d38b582e4094d57ad1f4116d5f9200", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 235 - }, - { - "Annotations": null, - "Digest": "sha256:49c6a3bffba4db85f0c178739267de92967fd1cbe66351f0ad71b0794a54f1dd", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 672 - }, - { - "Annotations": null, - "Digest": "sha256:5323de1a26cb18ab72836a586f43f7ace71d4c3ae1d32bbdb605dbca24a1b2d5", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 290 - }, - { - "Annotations": null, - "Digest": "sha256:110dfbac5781536633c8f4db803ac2d312635c138b6271950180d98edb25ad92", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 422 - }, - { - "Annotations": null, - "Digest": "sha256:6459549ab2f0a5cbcd9459b7c0dd9d84d781e6a54de176bf7577eba0ef8fa717", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 423 - }, - { - "Annotations": null, - "Digest": "sha256:064a69e5aa97e71c2a051b299f884d0d7cc5eb3a9874f701715196c689b6b0d9", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 900732 - }, - { - "Annotations": null, - "Digest": "sha256:ab9faf067017641a6a1a0cd5ca0a7822dce53cdcea43cc99935fa5782a504382", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 1525922 - }, - { - "Annotations": null, - "Digest": "sha256:ab8cea428fbebc365aa1f17ebb2cd216c71125d12451f07ed08a20042bb95f2a", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 54659 - }, - { - "Annotations": null, - "Digest": "sha256:3b5c004c51ee98510d7d0db263d915bb7315ee5f24d63ebdc6f67c704e3e2aa6", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 958 - }, - { - "Annotations": null, - "Digest": "sha256:82637563e123b17d0367abf543bdb5c464e3fe9be441a562c5b740998f6eed54", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 117 - }, - { - "Annotations": null, - "Digest": "sha256:d1b66950bec6912c850243f1d5b32e80dffc34df5885f7586ae7cf69d3aba78a", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 214 - }, - { - "Annotations": null, - "Digest": "sha256:556248b39507bff8be8a1d0b871d1460ef1125c270f671310aec774f90cc7fbb", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 455 - }, - { - "Annotations": null, - "Digest": "sha256:619208f024fb9939ea6a8a935a539f985880e9cec47c7684a5850530c965d7fd", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 321 - }, - { - "Annotations": null, - "Digest": "sha256:eb082d593603d8b87ed85cbfe5ce125325f46ac97fc2c468c4ae8fc6b46a5c5e", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 42812 - }, - { - "Annotations": null, - "Digest": "sha256:c385fbe872dfaf4fb44c7cbd9e67b093ef1eb5abae5ae4693c21808b8533043e", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 1434 - }, - { - "Annotations": null, - "Digest": "sha256:dd66dc57205d4b79bccdfaf5c0663dbe7e285265a43d6bd21116ff996abfd7c8", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 52666978 - }, - { - "Annotations": null, - "Digest": "sha256:6360eea5a3c80db3710e24f12acd00c8bc642d9201910b8eb89a9729a2a5760f", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 6531985 - }, - { - "Annotations": null, - "Digest": "sha256:2b62dc547d56dba81099f4713a8f35b3532d16f527e5068dadc7c7a0d06b7fc2", - "MIMEType": "application/vnd.docker.image.rootfs.diff.tar.gzip", - "Size": 183 - } - ], - "Name": "test.lagoon.sh/projectname/envname/service", - "Os": "linux", - "RepoTags": [ - "latest" - ] -} \ No newline at end of file From 2c80ddef24d69320cb71ca3d098c59d56138ef9d Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 9 Feb 2024 08:12:24 +1300 Subject: [PATCH 18/27] Factors out the actual processing from filter processImageInspectInsightsData into separate function --- internal/handler/imageInspectParserFilter.go | 48 +++++++++++--------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/internal/handler/imageInspectParserFilter.go b/internal/handler/imageInspectParserFilter.go index b61d0a2..820c277 100644 --- a/internal/handler/imageInspectParserFilter.go +++ b/internal/handler/imageInspectParserFilter.go @@ -27,37 +27,43 @@ func processImageInspectInsightsData(h *Messaging, insights InsightsData, v stri source := fmt.Sprintf("insights:image:%s", resource.Service) logger := slog.With("ProjectName", resource.Project, "EnvironmentName", resource.Environment, "Source", source) if insights.InsightsType == Image { - decoded, err := decodeGzipString(v) - if err != nil { - return nil, "", err - } _, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { return nil, "", apiErr } + environmentId := environment.Id - marshallDecoded, err := json.Marshal(decoded) - var imageInspect ImageData + return ProcessImageInspectData(v, logger, environmentId, source) + } + return []LagoonFact{}, "", nil +} - err = json.Unmarshal(marshallDecoded, &imageInspect) - if err != nil { - return nil, "", err - } +func ProcessImageInspectData(v string, logger *slog.Logger, environmentId int, source string) ([]LagoonFact, string, error) { + decoded, err := decodeGzipString(v) + if err != nil { + return nil, "", err + } - facts, err := processFactsFromImageInspect(logger, imageInspect, environment.Id, source) - if err != nil { - return nil, "", err - } - logger.Info("Successfully decoded image-inspect") - facts, err = KeyFactsFilter(facts) - if err != nil { - return nil, "", err - } + marshallDecoded, err := json.Marshal(decoded) + var imageInspect ImageData - return facts, source, nil + err = json.Unmarshal(marshallDecoded, &imageInspect) + if err != nil { + return nil, "", err } - return []LagoonFact{}, "", nil + + facts, err := processFactsFromImageInspect(logger, imageInspect, environmentId, source) + if err != nil { + return nil, "", err + } + logger.Info("Successfully decoded image-inspect") + facts, err = KeyFactsFilter(facts) + if err != nil { + return nil, "", err + } + + return facts, source, nil } func processFactsFromImageInspect(logger *slog.Logger, imageInspectData ImageData, id int, source string) ([]LagoonFact, error) { From d6731ad4217c3a980b94b68ca65fb2aa0b1f878c Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 9 Feb 2024 10:11:52 +1300 Subject: [PATCH 19/27] Factoring out further filter core functionality --- internal/handler/insightsFactsParserFilter.go | 47 ++++++------ internal/handler/insightsParserFilter.go | 75 +++++++++++-------- 2 files changed, 66 insertions(+), 56 deletions(-) diff --git a/internal/handler/insightsFactsParserFilter.go b/internal/handler/insightsFactsParserFilter.go index b2946a1..0343188 100644 --- a/internal/handler/insightsFactsParserFilter.go +++ b/internal/handler/insightsFactsParserFilter.go @@ -19,35 +19,36 @@ func processFactsInsightsData(h *Messaging, insights InsightsData, v string, api source := fmt.Sprintf("insights:facts:%s", resource.Service) logger := slog.With("ProjectName", resource.Project, "EnvironmentName", resource.Environment, "Source", source) if insights.LagoonType == Facts && insights.InsightsType == Raw { - r := strings.NewReader(v) + return ProcessFactsData(v, logger, source) + } + return nil, "", nil +} - // Decode base64 - //dec := base64.NewDecoder(base64.StdEncoding, r) - res, err := ioutil.ReadAll(r) - if err != nil { - slog.Error("Error reading insights data", "Error", err) - } +func ProcessFactsData(v string, logger *slog.Logger, source string) ([]LagoonFact, string, error) { + r := strings.NewReader(v) + // Decode base64 + res, err := ioutil.ReadAll(r) + if err != nil { + slog.Error("Error reading insights data", "Error", err) + } - facts, err := processFactsFromJSON(logger, res, source) - if err != nil { - return nil, "", err - } + facts, err := processFactsFromJSON(logger, res, source) + if err != nil { + return nil, "", err + } - facts, err = KeyFactsFilter(facts) - if err != nil { - return nil, "", err - } + facts, err = KeyFactsFilter(facts) + if err != nil { + return nil, "", err + } - if len(facts) == 0 { - return nil, "", fmt.Errorf("no facts to process") - } + if len(facts) == 0 { + return nil, "", fmt.Errorf("no facts to process") + } - //log.Printf("Successfully processed %d fact(s), for '%s:%s', from source '%s'", len(facts), resource.Project, resource.Environment, source) - logger.Info("Successfully processed facts", "number", len(facts)) + logger.Info("Successfully processed facts", "number", len(facts)) - return facts, source, nil - } - return nil, "", nil + return facts, source, nil } func processFactsFromJSON(logger *slog.Logger, facts []byte, source string) ([]LagoonFact, error) { diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index 5db42a7..bf4aab1 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -23,39 +23,9 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC return []LagoonFact{}, "", nil } - bom := new(cdx.BOM) - - // Decode base64 - r := strings.NewReader(v) - dec := base64.NewDecoder(base64.StdEncoding, r) - - res, err := ioutil.ReadAll(dec) + bom, err := getBOMfromPayload(v) if err != nil { - return nil, "", err - } - - fileType := http.DetectContentType(res) - - if fileType != "application/zip" && fileType != "application/x-gzip" && fileType != "application/gzip" { - decoder := cdx.NewBOMDecoder(bytes.NewReader(res), cdx.BOMFileFormatJSON) - if err = decoder.Decode(bom); err != nil { - return nil, "", err - } - } else { - // Compressed cyclonedx sbom - result, decErr := decodeGzipString(v) - if decErr != nil { - return nil, "", decErr - } - b, mErr := json.MarshalIndent(result, "", " ") - if mErr != nil { - return nil, "", mErr - } - - decoder := cdx.NewBOMDecoder(bytes.NewReader(b), cdx.BOMFileFormatJSON) - if err = decoder.Decode(bom); err != nil { - return nil, "", err - } + return []LagoonFact{}, "", err } // Determine lagoon resource destination @@ -65,7 +35,7 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC } // we process the SBOM here - + // TODO: This should actually live in its own function somewhere else. if h.ProblemsFromSBOM == true { isAlive, err := IsTrivyServerIsAlive(h.TrivyServerEndpoint) if err != nil { @@ -103,6 +73,45 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC return facts, source, nil } +// getBOMfromPayload is used to extract a *cdx.BOM from an incoming payload +func getBOMfromPayload(v string) (*cdx.BOM, error) { + bom := new(cdx.BOM) + + // Decode base64 + r := strings.NewReader(v) + dec := base64.NewDecoder(base64.StdEncoding, r) + + res, err := ioutil.ReadAll(dec) + if err != nil { + return nil, err + } + + fileType := http.DetectContentType(res) + + if fileType != "application/zip" && fileType != "application/x-gzip" && fileType != "application/gzip" { + decoder := cdx.NewBOMDecoder(bytes.NewReader(res), cdx.BOMFileFormatJSON) + if err = decoder.Decode(bom); err != nil { + return nil, err + } + } else { + // Compressed cyclonedx sbom + result, decErr := decodeGzipString(v) + if decErr != nil { + return nil, decErr + } + b, mErr := json.MarshalIndent(result, "", " ") + if mErr != nil { + return nil, mErr + } + + decoder := cdx.NewBOMDecoder(bytes.NewReader(b), cdx.BOMFileFormatJSON) + if err = decoder.Decode(bom); err != nil { + return nil, err + } + } + return bom, nil +} + func processFactsFromSBOM(logger *slog.Logger, facts *[]cdx.Component, environmentId int, source string) []LagoonFact { var factsInput []LagoonFact if len(*facts) == 0 { From bafef36a5f32ca912c8917856c8a0afdb71c8e05 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 9 Feb 2024 11:46:28 +1300 Subject: [PATCH 20/27] Moves bom decoder to its own file --- internal/handler/bomutils.go | 50 ++++++++++++++++++++++++ internal/handler/insightsParserFilter.go | 48 +---------------------- 2 files changed, 51 insertions(+), 47 deletions(-) create mode 100644 internal/handler/bomutils.go diff --git a/internal/handler/bomutils.go b/internal/handler/bomutils.go new file mode 100644 index 0000000..84799ee --- /dev/null +++ b/internal/handler/bomutils.go @@ -0,0 +1,50 @@ +package handler + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "github.com/CycloneDX/cyclonedx-go" + "io/ioutil" + "net/http" + "strings" +) + +// getBOMfromPayload is used to extract a *cdx.BOM from an incoming payload +func getBOMfromPayload(v string) (*cyclonedx.BOM, error) { + bom := new(cyclonedx.BOM) + + // Decode base64 + r := strings.NewReader(v) + dec := base64.NewDecoder(base64.StdEncoding, r) + + res, err := ioutil.ReadAll(dec) + if err != nil { + return nil, err + } + + fileType := http.DetectContentType(res) + + if fileType != "application/zip" && fileType != "application/x-gzip" && fileType != "application/gzip" { + decoder := cyclonedx.NewBOMDecoder(bytes.NewReader(res), cyclonedx.BOMFileFormatJSON) + if err = decoder.Decode(bom); err != nil { + return nil, err + } + } else { + // Compressed cyclonedx sbom + result, decErr := decodeGzipString(v) + if decErr != nil { + return nil, decErr + } + b, mErr := json.MarshalIndent(result, "", " ") + if mErr != nil { + return nil, mErr + } + + decoder := cyclonedx.NewBOMDecoder(bytes.NewReader(b), cyclonedx.BOMFileFormatJSON) + if err = decoder.Decode(bom); err != nil { + return nil, err + } + } + return bom, nil +} diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index bf4aab1..993f13b 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -1,17 +1,10 @@ package handler import ( - "bytes" - "encoding/base64" - "encoding/json" "fmt" - "io/ioutil" - "log/slog" - "net/http" - "strings" - cdx "github.com/CycloneDX/cyclonedx-go" "github.com/Khan/genqlient/graphql" + "log/slog" ) func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) { @@ -73,45 +66,6 @@ func processSbomInsightsData(h *Messaging, insights InsightsData, v string, apiC return facts, source, nil } -// getBOMfromPayload is used to extract a *cdx.BOM from an incoming payload -func getBOMfromPayload(v string) (*cdx.BOM, error) { - bom := new(cdx.BOM) - - // Decode base64 - r := strings.NewReader(v) - dec := base64.NewDecoder(base64.StdEncoding, r) - - res, err := ioutil.ReadAll(dec) - if err != nil { - return nil, err - } - - fileType := http.DetectContentType(res) - - if fileType != "application/zip" && fileType != "application/x-gzip" && fileType != "application/gzip" { - decoder := cdx.NewBOMDecoder(bytes.NewReader(res), cdx.BOMFileFormatJSON) - if err = decoder.Decode(bom); err != nil { - return nil, err - } - } else { - // Compressed cyclonedx sbom - result, decErr := decodeGzipString(v) - if decErr != nil { - return nil, decErr - } - b, mErr := json.MarshalIndent(result, "", " ") - if mErr != nil { - return nil, mErr - } - - decoder := cdx.NewBOMDecoder(bytes.NewReader(b), cdx.BOMFileFormatJSON) - if err = decoder.Decode(bom); err != nil { - return nil, err - } - } - return bom, nil -} - func processFactsFromSBOM(logger *slog.Logger, facts *[]cdx.Component, environmentId int, source string) []LagoonFact { var factsInput []LagoonFact if len(*facts) == 0 { From 352e77e5e1c7a695f33af95d5392e5d048258df3 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Fri, 9 Feb 2024 11:48:51 +1300 Subject: [PATCH 21/27] Separates out fact gathering to fact writing --- internal/handler/main.go | 67 +++++++++++++++++++++++------------ internal/handler/messaging.go | 17 +++++++-- 2 files changed, 60 insertions(+), 24 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index d7528f8..f20fabf 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -284,25 +284,22 @@ func (t *authedTransport) RoundTrip(req *http.Request) (*http.Response, error) { type LagoonSourceFactMap map[string][]LagoonFact // Incoming payload may contain facts or problems, so we need to handle these differently -func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource ResourceDestination, insights InsightsData) (err error) { +func (h *Messaging) gatherFactsFromInsightData(incoming *InsightsMessage, resource ResourceDestination, insights InsightsData) ([]LagoonSourceFactMap, error) { apiClient := h.getApiClient() + // Here we collect all source fact maps before writing them _once_ + lagoonSourceFactMapCollection := []LagoonSourceFactMap{} if resource.Project == "" && resource.Environment == "" { - return fmt.Errorf("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)") + return lagoonSourceFactMapCollection, fmt.Errorf("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)") } if insights.InputPayload == Payload && insights.LagoonType == Facts { for _, p := range incoming.Payload { lagoonSourceFactMap, err := parserFilterLoopForPayloads(insights, p, h, apiClient, resource) if err != nil { - return err - } - for sourceName, facts := range lagoonSourceFactMap { - err = sendResultsetToLagoon(facts, err, h, apiClient, resource, sourceName) - if err != nil { - return err - } + return lagoonSourceFactMapCollection, err } + lagoonSourceFactMapCollection = append(lagoonSourceFactMapCollection, lagoonSourceFactMap) } } @@ -310,18 +307,13 @@ func (h *Messaging) sendToLagoonAPI(incoming *InsightsMessage, resource Resource for _, p := range incoming.BinaryPayload { lagoonSourceFactMap, err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) if err != nil { - return err - } - for sourceName, facts := range lagoonSourceFactMap { - err = sendResultsetToLagoon(facts, err, h, apiClient, resource, sourceName) - if err != nil { - return err - } + return lagoonSourceFactMapCollection, err } + lagoonSourceFactMapCollection = append(lagoonSourceFactMapCollection, lagoonSourceFactMap) } } - return nil + return lagoonSourceFactMapCollection, nil } func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) (LagoonSourceFactMap, error) { @@ -360,8 +352,39 @@ func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messa return lagoonSourceFactMap, nil } +func trivySBOMProcessing(apiClient graphql.Client, trivyServerEndpoint string, resource ResourceDestination, payload string) error { + + bom, err := getBOMfromPayload(payload) + if err != nil { + return err + } + + // Determine lagoon resource destination + _, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) + if apiErr != nil { + return apiErr + } + + // we process the SBOM here + // TODO: This should actually live in its own function somewhere else. + isAlive, err := IsTrivyServerIsAlive(trivyServerEndpoint) + if err != nil { + return fmt.Errorf("trivy server not alive: %v", err.Error()) + } else { + slog.Debug("Trivy is reachable") + } + if isAlive { + err = SbomToProblems(apiClient, trivyServerEndpoint, "/tmp/", environment.Id, resource.Service, *bom) + } + if err != nil { + return err + } + return nil +} + // sendResultsetToLagoon will send results as facts to the lagoon api after processing via a parser filter -func sendResultsetToLagoon(result []LagoonFact, err error, h *Messaging, apiClient graphql.Client, resource ResourceDestination, source string) error { +func (h *Messaging) SendResultsetToLagoon(result []LagoonFact, resource ResourceDestination, source string) error { + apiClient := h.getApiClient() project, environment, apiErr := determineResourceFromLagoonAPI(apiClient, resource) if apiErr != nil { slog.Error(apiErr.Error()) @@ -376,10 +399,10 @@ func sendResultsetToLagoon(result []LagoonFact, err error, h *Messaging, apiClie return apiErr } - err = h.sendFactsToLagoonAPI(result, apiClient, resource, source) - if err != nil { - slog.Error("Error sending facts to Lagoon API", "error", err.Error()) - return err + e := h.sendFactsToLagoonAPI(result, apiClient, resource, source) + if e != nil { + slog.Error("Error sending facts to Lagoon API", "error", e.Error()) + return e } return nil diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index 8e8a3d6..d56b649 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -135,13 +135,26 @@ func (h *Messaging) processMessageQueue(message mq.Message) { insights.InsightsType != Direct { slog.Error("only 'sbom', 'direct', 'raw', and 'image' types are currently supported for api processing") } else { - err := h.sendToLagoonAPI(incoming, resource, insights) + lagoonSourceFactMapCollection, err := h.gatherFactsFromInsightData(incoming, resource, insights) if err != nil { - slog.Error("Unable to send to the API", "Error", err.Error()) + slog.Error("Unable to gather facts from incoming data", "Error", err.Error()) rejectMessage(false) return } + + // Here we actually go ahead and write all the facts with their source + for _, lsfm := range lagoonSourceFactMapCollection { + for sourceName, facts := range lsfm { + err := h.SendResultsetToLagoon(facts, resource, sourceName) + if err != nil { + slog.Error("Unable to write facts to api", "Error", err.Error()) + rejectMessage(false) + return + } + } + } + } } acknowledgeMessage() From d92ce3ba79fb83201fce5be28aad9268b3519dd7 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 12 Feb 2024 06:16:58 +1300 Subject: [PATCH 22/27] Deprecates uncompressed sbom and image types --- internal/handler/messaging.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index d56b649..ae873a0 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -206,9 +206,13 @@ func preprocessIncomingMessageData(incoming *InsightsMessage) (ResourceDestinati // Define insights type from incoming 'insightsType' label if insights.InputType != "" { switch insights.InputType { - case "sbom", "sbom-gz": + case "sbom": + return resource, insights, fmt.Errorf("insightsType of 'sbom' is deprecated, expect 'image-gz' - will not process") + case "sbom-gz": insights.InsightsType = Sbom - case "image", "image-gz": + case "image": + return resource, insights, fmt.Errorf("insightsType of 'image' is deprecated, expect 'image-gz' - will not process") + case "image-gz": insights.InsightsType = Image case "direct": insights.InsightsType = Direct From 11681c7f00b8aac2bf438d5f7142fd6fb5ff52b6 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 12 Feb 2024 07:14:24 +1300 Subject: [PATCH 23/27] Deprecates 'Direct' insights facts parser filter --- internal/handler/insightsFactsParserFilter.go | 100 -------------- .../handler/insightsFactsParserFilter_test.go | 124 ------------------ internal/handler/messaging.go | 2 +- .../testassets/rawFactsInsightsPayload.json | 40 ------ 4 files changed, 1 insertion(+), 265 deletions(-) delete mode 100644 internal/handler/insightsFactsParserFilter.go delete mode 100644 internal/handler/insightsFactsParserFilter_test.go delete mode 100644 internal/handler/testassets/rawFactsInsightsPayload.json diff --git a/internal/handler/insightsFactsParserFilter.go b/internal/handler/insightsFactsParserFilter.go deleted file mode 100644 index 0343188..0000000 --- a/internal/handler/insightsFactsParserFilter.go +++ /dev/null @@ -1,100 +0,0 @@ -package handler - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log/slog" - "strings" - - "github.com/Khan/genqlient/graphql" -) - -type FactsPayload struct { - Facts []LagoonFact `json:"facts,omitempty"` -} - -// Processes facts from insights payloads that come from reconcilled kubernetes payloads (e.g. include labels/annotations and compressed/encoded data) -func processFactsInsightsData(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) { - source := fmt.Sprintf("insights:facts:%s", resource.Service) - logger := slog.With("ProjectName", resource.Project, "EnvironmentName", resource.Environment, "Source", source) - if insights.LagoonType == Facts && insights.InsightsType == Raw { - return ProcessFactsData(v, logger, source) - } - return nil, "", nil -} - -func ProcessFactsData(v string, logger *slog.Logger, source string) ([]LagoonFact, string, error) { - r := strings.NewReader(v) - // Decode base64 - res, err := ioutil.ReadAll(r) - if err != nil { - slog.Error("Error reading insights data", "Error", err) - } - - facts, err := processFactsFromJSON(logger, res, source) - if err != nil { - return nil, "", err - } - - facts, err = KeyFactsFilter(facts) - if err != nil { - return nil, "", err - } - - if len(facts) == 0 { - return nil, "", fmt.Errorf("no facts to process") - } - - logger.Info("Successfully processed facts", "number", len(facts)) - - return facts, source, nil -} - -func processFactsFromJSON(logger *slog.Logger, facts []byte, source string) ([]LagoonFact, error) { - var factsInput []LagoonFact - - var factsPayload FactsPayload - err := json.Unmarshal(facts, &factsPayload) - if err != nil { - return factsInput, err - } - - if len(factsPayload.Facts) == 0 { - return factsInput, nil - } - - var filteredFacts []LagoonFact - keyFactsExistMap := make(map[string]bool) - - // Filter key facts - for _, v := range factsPayload.Facts { - if _, ok := keyFactsExistMap[v.Name]; !ok { - keyFactsExistMap[v.Name] = true - filteredFacts = append(filteredFacts, v) - } - } - - for _, f := range filteredFacts { - fact := LagoonFact{ - Environment: f.Environment, - Name: f.Name, - Value: f.Value, - Source: source, - Description: f.Description, - KeyFact: f.KeyFact, - Type: FactTypeText, - } - logger.Debug("Processing fact", "name", f.Name, "value", f.Value) - fact, err = ProcessLagoonFactAgainstRegisteredFilters(fact, f) - if err != nil { - return factsInput, err - } - factsInput = append(factsInput, fact) - } - return factsInput, nil -} - -func init() { - RegisterParserFilter(processFactsInsightsData) -} diff --git a/internal/handler/insightsFactsParserFilter_test.go b/internal/handler/insightsFactsParserFilter_test.go deleted file mode 100644 index 1f16cff..0000000 --- a/internal/handler/insightsFactsParserFilter_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package handler - -import ( - "encoding/json" - "io" - "io/ioutil" - "log" - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/Khan/genqlient/graphql" - "github.com/cheshir/go-mq" -) - -func Test_processFactsInsightsData(t *testing.T) { - type args struct { - h *Messaging - insights InsightsData - v string - apiClient graphql.Client - resource ResourceDestination - } - - h := Messaging{ - Config: mq.Config{}, - LagoonAPI: LagoonAPI{ - Endpoint: "http://localhost:3000/graphql", - }, - } - apiClient := h.getApiClient() - - testResponse, err := ioutil.ReadFile("./testassets/rawFactsInsightsPayload.json") - if err != nil { - t.Fatalf("Could not open file") - } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/" { - t.Errorf("Expected to request '/fixedvalue', got: %s", r.URL.Path) - } - w.WriteHeader(http.StatusOK) - w.Write(testResponse) - })) - defer server.Close() - - resp, err := http.Get(server.URL) - if err != nil { - panic(err) - } - defer resp.Body.Close() - - b, err := io.ReadAll(resp.Body) - if err != nil { - log.Fatalln(err) - } - - incoming := &InsightsMessage{} - json.Unmarshal(b, incoming) - - var payload PayloadInput - for _, p := range incoming.Payload { - payload = p - } - - data, err := json.Marshal(payload) - if err != nil { - panic(err) - } - - var tests = []struct { - name string - args args - want []LagoonFact - want1 string - wantErr bool - }{ - { - name: "raw facts insights payload", - args: args{ - h: &Messaging{}, - insights: InsightsData{ - InputPayload: Payload, - InsightsType: Raw, - LagoonType: Facts, - }, - v: string(data), - apiClient: apiClient, - resource: ResourceDestination{ - Project: "lagoon-demo-org", - Service: "cli", - }, - }, - want: []LagoonFact{ - { - Environment: 3, - Name: "drupal-core", - Value: "9.0.1", - Source: "insights:facts:cli", - Description: "Drupal CMS version found on environment", - KeyFact: true, - Type: "TEXT", - }, - }, - want1: "insights:facts:cli", - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, got1, err := processFactsInsightsData(tt.args.h, tt.args.insights, tt.args.v, tt.args.apiClient, tt.args.resource) - if (err != nil) != tt.wantErr { - t.Errorf("processFactsInsightsData() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("processFactsInsightsData() got = %v, want %v", got, tt.want) - } - if got1 != tt.want1 { - t.Errorf("processFactsInsightsData() got1 = %v, want %v", got1, tt.want1) - } - }) - } -} diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index ae873a0..506c255 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -215,7 +215,7 @@ func preprocessIncomingMessageData(incoming *InsightsMessage) (ResourceDestinati case "image-gz": insights.InsightsType = Image case "direct": - insights.InsightsType = Direct + return resource, insights, fmt.Errorf("insightsType of 'direct' is deprecated, expect 'direct.facts' - will not process") default: insights.InsightsType = Raw } diff --git a/internal/handler/testassets/rawFactsInsightsPayload.json b/internal/handler/testassets/rawFactsInsightsPayload.json deleted file mode 100644 index 564fa66..0000000 --- a/internal/handler/testassets/rawFactsInsightsPayload.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "payload": [ - { - "project": "lagoon-demo-org", - "environment": "main", - "facts": [ - { - "name": "drupal-core", - "value": "9.0.1", - "environment": 3, - "source": "insights:facts:cli", - "description": "Drupal CMS version found on environment", - "category": "Framework", - "keyFact": true, - "type": "TEXT" - }, - { - "name": "php-version", - "value": "8.0.3", - "environment": 3, - "source": "insights:facts:cli", - "description": "PHP version found on environment", - "category": "Programming language", - "keyFact": false, - "type": "TEXT" - } - ] - } - ], - "binaryPayload": {}, - "annotations": {}, - "labels": { - "lagoon.sh/project": "lagoon-demo-org", - "lagoon.sh/environment": "main", - "lagoon.sh/service": "cli", - "lagoon.sh/insightsType": "raw", - "lagoon.sh/insightsOutputFileMIMEType": "application/json", - "lagoon.sh/insightsOutputFileExt": "json" - } -} From 65d08112ed3cccd682c6f73264af9d293180aa4d Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 12 Feb 2024 09:15:53 +1300 Subject: [PATCH 24/27] Removes parser filter loop and support for non-binary payloads --- internal/handler/main.go | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/internal/handler/main.go b/internal/handler/main.go index f20fabf..6cdc637 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -293,16 +293,6 @@ func (h *Messaging) gatherFactsFromInsightData(incoming *InsightsMessage, resour return lagoonSourceFactMapCollection, fmt.Errorf("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)") } - if insights.InputPayload == Payload && insights.LagoonType == Facts { - for _, p := range incoming.Payload { - lagoonSourceFactMap, err := parserFilterLoopForPayloads(insights, p, h, apiClient, resource) - if err != nil { - return lagoonSourceFactMapCollection, err - } - lagoonSourceFactMapCollection = append(lagoonSourceFactMapCollection, lagoonSourceFactMap) - } - } - if insights.InputPayload == BinaryPayload && insights.LagoonType == Facts { for _, p := range incoming.BinaryPayload { lagoonSourceFactMap, err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) @@ -330,28 +320,6 @@ func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messa return lagoonSourceFactMap, nil } -func parserFilterLoopForPayloads(insights InsightsData, p PayloadInput, h *Messaging, apiClient graphql.Client, resource ResourceDestination) (LagoonSourceFactMap, error) { - lagoonSourceFactMap := LagoonSourceFactMap{} - for _, filter := range parserFilters { - var result []LagoonFact - var source string - - json, err := json.Marshal(p) - if err != nil { - slog.Error("Error marshalling data", "error", err.Error()) - return lagoonSourceFactMap, err - } - - result, source, err = filter(h, insights, fmt.Sprintf("%s", json), apiClient, resource) - if err != nil { - slog.Error("Error Filtering payload", "error", err.Error()) - return lagoonSourceFactMap, err - } - lagoonSourceFactMap[source] = result - } - return lagoonSourceFactMap, nil -} - func trivySBOMProcessing(apiClient graphql.Client, trivyServerEndpoint string, resource ResourceDestination, payload string) error { bom, err := getBOMfromPayload(payload) From 5c77fa8c31db0c371859797469daa730967de4db Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Mon, 12 Feb 2024 14:06:27 +1300 Subject: [PATCH 25/27] Removes dynamic message filter logic and adds comments --- internal/handler/filters.go | 2 + internal/handler/imageInspectParserFilter.go | 7 +-- internal/handler/insightsParserFilter.go | 4 -- internal/handler/main.go | 45 ++++++++++++-------- internal/handler/messaging.go | 2 +- internal/handler/parserfilter.go | 19 +-------- internal/handler/registerFilters.go | 7 +++ 7 files changed, 41 insertions(+), 45 deletions(-) diff --git a/internal/handler/filters.go b/internal/handler/filters.go index 58e1b30..f67fdf5 100644 --- a/internal/handler/filters.go +++ b/internal/handler/filters.go @@ -1,5 +1,7 @@ package handler +// KeyFactsFilter simply takes a slice of LagoonFacts and filters out anything not marked with KeyFact = true +// This is used downstream to ensure only key facts are written to the DB func KeyFactsFilter(factsInput []LagoonFact) ([]LagoonFact, error) { var filteredFacts []LagoonFact for _, v := range factsInput { diff --git a/internal/handler/imageInspectParserFilter.go b/internal/handler/imageInspectParserFilter.go index 820c277..3b24a92 100644 --- a/internal/handler/imageInspectParserFilter.go +++ b/internal/handler/imageInspectParserFilter.go @@ -76,6 +76,7 @@ func processFactsFromImageInspect(logger *slog.Logger, imageInspectData ImageDat // Check if image inspect contains useful environment variables if imageInspectData.Env != nil { for _, v := range imageInspectData.Env { + logger.Debug("Processing env data", "data", v) var envSplitStr = strings.Split(v, "=") env := EnvironmentVariable{ Key: envSplitStr[0], @@ -102,14 +103,10 @@ func processFactsFromImageInspect(logger *slog.Logger, imageInspectData ImageDat Type: FactTypeText, } - logger.Debug("Processing fact", "name", f.Key, "value", f.Value) + logger.Debug("Processing environment fact", "name", f.Key, "value", f.Value) fact, _ = ProcessLagoonFactAgainstRegisteredFilters(fact, f) factsInput = append(factsInput, fact) } return factsInput, nil } - -func init() { - RegisterParserFilter(processImageInspectInsightsData) -} diff --git a/internal/handler/insightsParserFilter.go b/internal/handler/insightsParserFilter.go index 993f13b..8c3de48 100644 --- a/internal/handler/insightsParserFilter.go +++ b/internal/handler/insightsParserFilter.go @@ -105,7 +105,3 @@ func processFactsFromSBOM(logger *slog.Logger, facts *[]cdx.Component, environme } return factsInput } - -func init() { - RegisterParserFilter(processSbomInsightsData) -} diff --git a/internal/handler/main.go b/internal/handler/main.go index 6cdc637..1bf13a4 100644 --- a/internal/handler/main.go +++ b/internal/handler/main.go @@ -293,31 +293,42 @@ func (h *Messaging) gatherFactsFromInsightData(incoming *InsightsMessage, resour return lagoonSourceFactMapCollection, fmt.Errorf("no resource definition labels could be found in payload (i.e. lagoon.sh/project or lagoon.sh/environment)") } - if insights.InputPayload == BinaryPayload && insights.LagoonType == Facts { + slog.Debug("Processing data", "InputPayload", insights.InputPayload, "LagoonType", insights.LagoonType, "InsightsType", insights.InsightsType) + + if insights.InputPayload == BinaryPayload { + var binaryPayload string + // We simply pop off the first item - this drastically simplifies the logic below for _, p := range incoming.BinaryPayload { - lagoonSourceFactMap, err := parserFilterLoopForBinaryPayloads(insights, p, h, apiClient, resource) + binaryPayload = p + break + } + lagoonSourceFactMap := LagoonSourceFactMap{} + // since we only have two parser filter types now - let's explicitly call them + + // First we call the image inspect processor, in case there's anything there + if insights.InsightsType == Image { + slog.Debug("Running InsightsType == Image. Got insights image input") + result, source, err := processImageInspectInsightsData(h, insights, binaryPayload, apiClient, resource) if err != nil { - return lagoonSourceFactMapCollection, err + slog.Error("Error running filter", "error", err.Error()) } + lagoonSourceFactMap[source] = result lagoonSourceFactMapCollection = append(lagoonSourceFactMapCollection, lagoonSourceFactMap) } - } - - return lagoonSourceFactMapCollection, nil -} -func parserFilterLoopForBinaryPayloads(insights InsightsData, p string, h *Messaging, apiClient graphql.Client, resource ResourceDestination) (LagoonSourceFactMap, error) { - lagoonSourceFactMap := LagoonSourceFactMap{} - for _, filter := range parserFilters { + // Then we call the SBOM processor, in case we're dealing with this type + if insights.InsightsType == Sbom { + result, source, err := processSbomInsightsData(h, insights, binaryPayload, apiClient, resource) + if err != nil { + slog.Error("Error running filter", "error", err.Error()) + } + lagoonSourceFactMap[source] = result - result, source, err := filter(h, insights, p, apiClient, resource) - if err != nil { - slog.Error("Error running filter", "error", err.Error()) - return lagoonSourceFactMap, err } - lagoonSourceFactMap[source] = result + lagoonSourceFactMapCollection = append(lagoonSourceFactMapCollection, lagoonSourceFactMap) } - return lagoonSourceFactMap, nil + + return lagoonSourceFactMapCollection, nil } func trivySBOMProcessing(apiClient graphql.Client, trivyServerEndpoint string, resource ResourceDestination, payload string) error { @@ -334,7 +345,6 @@ func trivySBOMProcessing(apiClient graphql.Client, trivyServerEndpoint string, r } // we process the SBOM here - // TODO: This should actually live in its own function somewhere else. isAlive, err := IsTrivyServerIsAlive(trivyServerEndpoint) if err != nil { return fmt.Errorf("trivy server not alive: %v", err.Error()) @@ -463,6 +473,7 @@ func (h *Messaging) sendToLagoonS3(incoming *InsightsMessage, insights InsightsD slog.Info(fmt.Sprintf("Successfully created %s", h.S3Config.Bucket)) } + // TODO: this can likely be removed if len(incoming.Payload) != 0 { b, err := json.Marshal(incoming) if err != nil { diff --git a/internal/handler/messaging.go b/internal/handler/messaging.go index 506c255..836877a 100644 --- a/internal/handler/messaging.go +++ b/internal/handler/messaging.go @@ -117,7 +117,7 @@ func (h *Messaging) processMessageQueue(message mq.Message) { slog.Debug("Insights", "data", fmt.Sprint(insights)) slog.Debug("Target", "data", fmt.Sprint(resource)) - // Process s3 upload + // Process s3 upload - that is, upload the incoming insights data to an s3 bucket if !h.S3Config.Disabled { if insights.InsightsType != Direct { err := h.sendToLagoonS3(incoming, insights, resource) diff --git a/internal/handler/parserfilter.go b/internal/handler/parserfilter.go index ab205c0..423879c 100644 --- a/internal/handler/parserfilter.go +++ b/internal/handler/parserfilter.go @@ -1,9 +1,6 @@ package handler -import ( - "github.com/Khan/genqlient/graphql" -) - +// a parserFilter will type parserFilter interface { isFilteredOut() bool hasError() bool @@ -15,17 +12,3 @@ type parserFilter interface { setFactField(fieldname string, value string) parserFilter getFact() LagoonFact } - -type ParserFilterFunc func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) - -var parserFilters []ParserFilterFunc - -func RegisterParserFilter(pf ParserFilterFunc) { - parserFilters = append(parserFilters, func(h *Messaging, insights InsightsData, v string, apiClient graphql.Client, resource ResourceDestination) ([]LagoonFact, string, error) { - result, source, err := pf(h, insights, v, apiClient, resource) - if err != nil { - return nil, "", err - } - return result, source, nil - }) -} diff --git a/internal/handler/registerFilters.go b/internal/handler/registerFilters.go index 01d5aab..4320897 100644 --- a/internal/handler/registerFilters.go +++ b/internal/handler/registerFilters.go @@ -33,10 +33,13 @@ type FactTransforms struct { Transforms []FactTransform `json:"transforms"` } +// registerFilter takes in a parserFilter defining function (capturing a transform programmatically) and adds it to +// the global list of parserFilters func registerFilter(filter func(filter parserFilter) parserFilter) { KeyFactFilters = append(KeyFactFilters, filter) } +// ProcessLagoonFactAgainstRegisteredFilters will take in a single fact and run it against all KeyFactFilters func ProcessLagoonFactAgainstRegisteredFilters(fact LagoonFact, insightsRawData interface{}) (LagoonFact, error) { for _, filter := range KeyFactFilters { pf := FactProcessor{ @@ -82,8 +85,12 @@ func LoadTransformsFromDisk(filename string) ([]FactTransform, error) { return ret.Transforms, nil } +// GenerateFilterFromTransform will take a transform description and generate a function that will check a fact +// against a transform - that is, see if we need to change its name/value. +// Perhaps the only tricky thing here is that func GenerateFilterFromTransform(transform FactTransform) (func(filter parserFilter) parserFilter, error) { + // we build and return a function that captures the given transform (description of changes) in a closure return func(filter parserFilter) parserFilter { if transform.Type != "" { From 12c32d1a0f21b1409dabfa85c6790574864e51c2 Mon Sep 17 00:00:00 2001 From: Blaize M Kaye Date: Tue, 13 Feb 2024 07:32:24 +1300 Subject: [PATCH 26/27] Updates Documentation --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 6e592a2..91ae614 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,64 @@ # Lagoon Insights Handler -This service will listen for messages and handle the requirements of the payload. +The Lagoon Insights Handler service is a primary hub for processing incoming data about Lagoon environments. +These `insights` are gathered in remote clusters via the [insights-remote](https://github.com/uselagoon/insights-remote) service, +sent to the present service via the [Lagoon broker](https://github.com/uselagoon/lagoon/tree/main/services/broker) (presently a RabbitMQ instance), and then processed. + + +## Data types processed by the Insights Handler + +Presently there are several data sources processed by the Insights Handler. + +1. Image inspect information + 2. This is, essentially, the output of a `docker inspect` run at a project's build deploy time. +4. SBOM processing + 5. The [Lagoon Build Deploy Tool](https://github.com/uselagoon/build-deploy-tool) will run trivy on the resulting images that make up a project. + 6. This produces a Software Bill of Materials, essentially a list of all packages installed on those images. + 7. These SBOMs are processed and, optionally, run through Trivy to produce a list of vulnerabilities, which are written to Lagoon as `Problems` +8. Direct `Fact` and `Problem` writes to Lagoon + 9. Effectively the core part of these write described in the section of the Insights Remote documentation about [Insights written directly to insights remote](https://github.com/uselagoon/insights-remote?tab=readme-ov-file#insights-written-directly-to-insights-remote). + +## Parsing, filtering, and transforming incoming data + +The Insights Handler allows one to set up processing of incoming data to manipulate it before it is sent to Lagoon as Facts. +This is achieved by specifying so-called "filter-transformers", which are defined as YAML and passed to the handler on startup. + +The assumption is that data coming from writes to the `insights remote` service endpoints will be formatted as one would like them to appear in the Lagoon API. +However, data from image inspect and SBOM generation processes are not under our control at generation time, therefore being able to manipulate them is useful. + +Let's take a look at the structure of a simple filter-transformer + +``` + - type: Package + lookupvalue: + - name: Name + value: alpine + exactMatch: true + transformations: + - name: Name + value: Alpine Linux + - name: Category + value: OS + - name: Description + value: Base image Alpine Linux version +``` + +The `type` describes the source that this transformer is concerned with - this can be one of three values: +- Package - targets SBOMs +- EnvironmentVariable - will target env vars in image inspect data (i.e. if an `ENV` is part of the resultant image) +- InspectLabel - targets labels listed in image inspect data + +The `lookupvalue` describes rules for matching the item in the incoming data. In the case above, we're looking for a Package `Name` that matches `alpine` exactly. + +The `transformations` section describes what we'd like the resulting fact's data to be - so instead of `Name` being `alpine` (which is what we see in our target rule), we replace `alpine` with the text `Alpine Linux`. + +There are several other examples in `default_filter_transformers.yaml`. + +### Specifying alternative filters and transformations + +The Insights Handler will load the default handlers found in the root directory `default_filter_transformers.yaml`. + +This can be overridden in serveral ways - either by replacing the file itself (in, for instance, a configMap, etc.), via the flag `--filter-transformer-file`, or the env var `FILTER_TRANSFORMER_FILE`. ## Facts Currently, the main purpose is to consume a Software Bill of Materials (SBOM) of facts from the logs queue, process @@ -9,28 +67,17 @@ and push to the api and s3 bucket. ## Local development +Assuming that you're running a Lagoon development instance (i.e. pulling the Lagoon source and running `make up`), some reasonable defaults for developing locally could be given by the following: + go run main.go \ -rabbitmq-username guest \ -rabbitmq-password guest \ -lagoon-api-host http://localhost:8888/graphql \ --jwt-token-signing-key secret \ --access-key-id minio \ - --secret-access-key minio123 + --secret-access-key minio123 \ + --debug=true -To compile GraphQL schema, type-safe structs and response data with genqlient we just add a query/mutation inside of `lagoonclient/genqlient.graphql` and run this: +To compile GraphQL schema, type-safe structs and response data with genqlient we just add a query/mutation inside of `lagoonclient/genqlient.graphql` and run: go generate - -## Configmap labels - -```json - "labels": { - "lagoon.sh/project": "lagoon", - "lagoon.sh/environment": "main", - "lagoon.sh/service": "cli", - "lagoon.sh/insightsType": ["sbom", "image"], - "lagoon.sh/insightsOutputCompressed": ["true", "false" (default)] (optional), - "lagoon.sh/insightsOutputFileExt": ["json (default)", "txt", "csv", "html", "jpg"] (optional), - "lagoon.sh/insightsOutputFileMIMEType": ["text/html", "image/svg+xml"] (optional) - } -``` \ No newline at end of file From d15c5b8b8fd1a67fb8d094f8718be2d0d82b7834 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 19:22:46 +0000 Subject: [PATCH 27/27] Update module github.com/docker/docker to v25 --- go.mod | 13 ++++++++++--- go.sum | 58 ++++++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 780cd6f..38b90ee 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,12 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/NeowayLabs/wabbit v0.0.0-20200409220312-12e68ab5b0c6 // indirect + github.com/containerd/containerd v1.7.13 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -24,27 +28,30 @@ require ( github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/moby/sys/mount v0.3.3 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/pborman/uuid v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/rs/xid v1.2.1 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vektah/gqlparser/v2 v2.5.10 // indirect golang.org/x/crypto v0.16.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.10.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) replace ( github.com/docker/cli => github.com/docker/cli v20.10.19+incompatible - github.com/docker/docker => github.com/docker/docker v20.10.19+incompatible + github.com/docker/docker => github.com/docker/docker v25.0.3+incompatible ) require ( diff --git a/go.sum b/go.sum index 5b3412a..1b3bac0 100644 --- a/go.sum +++ b/go.sum @@ -3,16 +3,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/CycloneDX/cyclonedx-go v0.7.2-0.20230625092137-07e2f29defc3 h1:NqeV+ZMqpcosu0Xg2VW14Ru9ayBs/toe2oihS7sN6Xo= -github.com/CycloneDX/cyclonedx-go v0.7.2-0.20230625092137-07e2f29defc3/go.mod h1:fGXSp1lCDfMQ8KR1EjxT4ewc5HHhGczRF2pWhLSWohs= github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/hcsshim v0.8.14 h1:lbPVK25c1cu5xTLITwpUcxoA9vKrKErASPYygvouJns= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -22,16 +20,18 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.7.13 h1:wPYKIeGMN8vaggSKuV1X0wZulpMz4CrgEsZdaCyB6Is= +github.com/containerd/containerd v1.7.13/go.mod h1:zT3up6yTRfEUa6+GsITYIJNgSVL9NQ4x4h1RPzk0Wu4= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= @@ -44,17 +44,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/docker/docker v20.10.19+incompatible h1:lzEmjivyNHFHMNAFLXORMBXyGIhw/UP4DvJwvyKYq64= -github.com/docker/docker v20.10.19+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ= +github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsouza/go-dockerclient v1.7.3 h1:i6iMcktl688vsKUEExA6gU1UjPgIvmGtJeQ0mbuFqZo= github.com/fsouza/go-dockerclient v1.7.3/go.mod h1:8xfZB8o9SptLNJ13VoV5pMiRbZGWkU/Omu5VOu/KC9Y= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -74,7 +81,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -111,12 +117,14 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= -github.com/moby/sys/mount v0.3.3 h1:fX1SVkXFJ47XWDoeFW4Sq7PdQJnV2QIDZAqjNqgEjUs= -github.com/moby/sys/mount v0.3.3/go.mod h1:PBaEorSNTLG5t/+4EgukEQVlAvVEc6ZjTySwKdqp5K0= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= +github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= +github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg= +github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -129,10 +137,10 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= @@ -160,8 +168,8 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -195,8 +203,15 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -208,6 +223,8 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -245,7 +262,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210216224549-f992740a1bac/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -265,6 +281,8 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= +golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=