Merge pull request #530 from solarwinds/feature/NH-69215-upgradeColle…

…ctorAndImages

NH-69215:  Upgrade OTEL Collector and 3rd party Docker images

.github/workflows/buildAndDeploy.yml

@@ -184,7 +184,7 @@ jobs:
       - name: Create Dockerfile with Helm repository
         run: |
           cat <<EOF > Dockerfile
-          FROM python:3.8-alpine
+          FROM python:3.12-alpine3.19
           WORKDIR /app
           COPY . .
           EXPOSE 5000

build/docker/Dockerfile

@@ -5,7 +5,7 @@ COPY ["./src/", "./src/"]
 FROM base as builder
 
 COPY /build/swi-k8s-opentelemetry-collector.yaml /src/swi-k8s-opentelemetry-collector.yaml
-RUN GO111MODULE=on go install go.opentelemetry.io/collector/cmd/builder@v0.81.0
+RUN GO111MODULE=on go install go.opentelemetry.io/collector/cmd/builder@v0.91.0
 RUN CGO_ENABLED=0 GOEXPERIMENT=boringcrypto /go/bin/builder --config ./swi-k8s-opentelemetry-collector.yaml --output-path ./
 
 # create vendor folder (for mend scanning)
@@ -16,7 +16,7 @@ RUN if [[ -z "$CREATE_VENDOR_DIR" ]] ; then echo vendor creation skipped ; else
 FROM base as tests
 RUN cd src/processor/swmetricstransformprocessor && go test ./...
 
-FROM debian:12.2 as journal
+FROM debian:12.4 as journal
 RUN apt update
 RUN apt install -y systemd
 COPY /build/docker/copy-journalbinary.sh /script.sh
@@ -27,7 +27,7 @@ FROM base as wrapper
 WORKDIR /src/src/wrapper
 RUN CGO_ENABLED=0 GOEXPERIMENT=boringcrypto go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /bin/wrapper && chmod +x /bin/wrapper
 
-FROM gcr.io/distroless/static@sha256:6706c73aae2afaa8201d63cc3dda48753c09bcd6c300762251065c0f7e602b25
+FROM gcr.io/distroless/static-debian12@sha256:4a2c1a51ae5e10ec4758a0f981be3ce5d6ac55445828463fce8dff3a355e0b75
 
 ARG USER_UID=10001
 USER ${USER_UID}

build/docker/Dockerfile.Windows-2022

@@ -1,11 +1,11 @@
-FROM docker.io/library/golang:1.21.5-nanoserver-ltsc2022@sha256:4a6bc1c595464b6cc4f916024390f88768481e17209425cd7fe2bace2fa429dc as base
+FROM docker.io/library/golang:1.21.5-nanoserver-ltsc2022@sha256:99e9d084a2d3e0e4d647b223db99c839ec66a098ec67a7d173af7feecacb2c44 as base
 WORKDIR /src
 COPY ["./src/", "./src/"]
 
 FROM base as builder
 
 COPY /build/swi-k8s-opentelemetry-collector.yaml /src/swi-k8s-opentelemetry-collector.yaml
-RUN go install go.opentelemetry.io/collector/cmd/builder@v0.81.0
+RUN go install go.opentelemetry.io/collector/cmd/builder@v0.91.0
 
 ARG CGO_ENABLED=0
 ARG GOEXPERIMENT=boringcrypto

build/docker/IntegrationTest.Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.12-alpine3.17
+FROM python:3.12-alpine3.19
 
 RUN apk add --update --no-cache curl ca-certificates bash
 

build/swi-k8s-opentelemetry-collector.yaml

@@ -1,51 +1,44 @@
 dist:
   name: swi-k8s-opentelemetry-collector
   description: "SolarWinds distribution for OpenTelemetry"
-  otelcol_version: "0.81.0"
+  otelcol_version: "0.91.0"
   version: "0.9.2"
 exporters:
-  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.81.0
+  - gomod: go.opentelemetry.io/collector/exporter/otlpexporter v0.91.0
 
 connectors:
-  - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.81.0
+  - gomod: go.opentelemetry.io/collector/connector/forwardconnector v0.91.0
 
 receivers:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8seventsreceiver v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/windowseventlogreceiver v0.81.0
-  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/receivercreator v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/simpleprometheusreceiver v0.81.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/k8seventsreceiver v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/filelogreceiver v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/windowseventlogreceiver v0.91.0
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/receivercreator v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/receiver/simpleprometheusreceiver v0.91.0
 
 processors:
-  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.81.0
-  - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.81.0
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.91.0
+  - gomod: go.opentelemetry.io/collector/processor/memorylimiterprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/groupbyattrsprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatorateprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/cumulativetodeltaprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/attributesprocessor v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.91.0
     path: "./src/processor/k8sattributesprocessor"
   - gomod: github.com/solarwinds/swi-k8s-opentelemetry-collector/processor/swmetricstransformprocessor v0.0.1
     path: "./src/processor/swmetricstransformprocessor"
 
 extensions:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.81.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.91.0
     import: "github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage"
-  - gomod: go.opentelemetry.io/collector/extension/ballastextension v0.81.0
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/k8sobserver v0.81.0
-
-replaces:
-  - github.com/jaegertracing/jaeger => github.com/jaegertracing/jaeger v1.47
-  - golang.org/x/net => golang.org/x/net v0.17.0
-  - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.333
-  - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp => go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0
-  - google.golang.org/grpc => google.golang.org/grpc v1.59.0
+  - gomod: go.opentelemetry.io/collector/extension/ballastextension v0.91.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/k8sobserver v0.91.0

deploy/helm/node-collector-config.yaml

@@ -294,14 +294,7 @@ connectors:
 
 receivers:
 {{- if and (not .isWindows) .Values.otel.logs.journal }}
-  journald/runlogs:
-    directory: /run/log/journal
-    units:
-      - kubelet
-      - docker
-      - containerd
-  journald/varlogs:
-    directory: /var/log/journal
+  journald:
     units:
       - kubelet
       - docker
@@ -554,8 +547,7 @@ service:
         - groupbyattrs/common-all
         - resource/journal
       receivers:
-        - journald/runlogs
-        - journald/varlogs
+        - journald
 {{- end }}
 
     logs:

deploy/helm/templates/autoupdate/autoupdate-cronjob.yaml

@@ -19,7 +19,7 @@ spec:
           {{- end }}
           containers:
           - name: helm-upgrade
-            image: "{{ include "common.image" (tuple . .Values.autoupdate (tuple "image" "autoupdate") "alpine/k8s:1.27.2") }}"
+            image: "{{ include "common.image" (tuple . .Values.autoupdate (tuple "image" "autoupdate") "alpine/k8s:1.27.8") }}"
             imagePullPolicy: {{ .Values.otel.image.pullPolicy }}
             command: 
               - /bin/bash

deploy/helm/templates/metrics-deployment.yaml

@@ -77,7 +77,7 @@ spec:
         {{- end }}
         {{- if .Values.otel.metrics.swi_endpoint_check }}
         - name: otel-endpoint-check
-          image: "{{ include "common.image" (tuple . .Values.otel.init_images "swi_endpoint_check" "fullstorydev/grpcurl:v1.8.7") }}"
+          image: "{{ include "common.image" (tuple . .Values.otel.init_images "swi_endpoint_check" "fullstorydev/grpcurl:v1.8.9") }}"
           imagePullPolicy: {{ .Values.otel.init_images.swi_endpoint_check.pullPolicy }}
           command: ['/bin/grpcurl', '-expand-headers',
                     '-proto', 'opentelemetry/proto/collector/logs/v1/logs_service.proto',

deploy/helm/tests/__snapshot__/metrics-deployment_test.yaml.snap

@@ -84,7 +84,7 @@ Metrics collector spec should match snapshot when using default values:
                 key: SOLARWINDS_API_TOKEN
                 name: solarwinds-api-token
                 optional: true
-        image: fullstorydev/grpcurl:v1.8.7
+        image: fullstorydev/grpcurl:v1.8.9
         imagePullPolicy: IfNotPresent
         name: otel-endpoint-check
         volumeMounts:

deploy/helm/tests/__snapshot__/node-collector-config-map_test.yaml.snap

@@ -970,14 +970,7 @@ Node collector config should match snapshot when fargate is enabled:
           poll_interval: 200ms
           start_at: end
           storage: file_storage/checkpoints
-        journald/runlogs:
-          directory: /run/log/journal
-          units:
-          - kubelet
-          - docker
-          - containerd
-        journald/varlogs:
-          directory: /var/log/journal
+        journald:
           units:
           - kubelet
           - docker
@@ -1035,8 +1028,7 @@ Node collector config should match snapshot when fargate is enabled:
             - groupbyattrs/common-all
             - resource/journal
             receivers:
-            - journald/runlogs
-            - journald/varlogs
+            - journald
           metrics/discovery:
             exporters:
             - forward/metric-exporter
@@ -2030,14 +2022,7 @@ Node collector config should match snapshot when using default values:
           poll_interval: 200ms
           start_at: end
           storage: file_storage/checkpoints
-        journald/runlogs:
-          directory: /run/log/journal
-          units:
-          - kubelet
-          - docker
-          - containerd
-        journald/varlogs:
-          directory: /var/log/journal
+        journald:
           units:
           - kubelet
           - docker
@@ -2132,8 +2117,7 @@ Node collector config should match snapshot when using default values:
             - groupbyattrs/common-all
             - resource/journal
             receivers:
-            - journald/runlogs
-            - journald/varlogs
+            - journald
           metrics:
             exporters:
             - otlp

deploy/helm/tests/metrics-deployment_test.yaml

@@ -61,7 +61,7 @@ tests:
     asserts:
     - equal:
           path: spec.template.spec.initContainers[0].image
-          value: fullstorydev/grpcurl:v1.8.7
+          value: fullstorydev/grpcurl:v1.8.9
   - it: Image otel-endpoint-check should be correct when overriden repository
     template: metrics-deployment.yaml
     set:
@@ -70,7 +70,7 @@ tests:
     asserts:
     - equal:
           path: spec.template.spec.initContainers[0].image
-          value: aws/test-grpcurl:v1.8.7
+          value: aws/test-grpcurl:v1.8.9
   - it: Image otel-endpoint-check should be correct when overriden tag
     template: metrics-deployment.yaml
     set:

doc/collectorPipeline.md

@@ -107,13 +107,9 @@ stateDiagram-v2
 
     nc_logsJournalPipeline: 'logs/journal' pipeline
     state nc_logsJournalPipeline {
-      nc_r3_1: 'journald/runlogs' receiver
-      nc_r3_2: 'journald/varlogs' receiver
-      state nc_p3 <<join>>
+      nc_r3: 'journald' receiver
       nc_e3: 'forward/logs-exporter' connector
-      nc_r3_1 --> nc_p3
-      nc_r3_2 --> nc_p3
-      nc_p3 --> nc_e3 : processors
+      nc_r3 --> nc_e3 : processors
     }
 
     nc_metricsPipeline: 'metrics' pipeline

doc/development.md

@@ -171,7 +171,7 @@ helm unittest -u deploy/helm
 Integration tests are located in `tests/integration` and are supposed to verify if metric processing is delivering expected outcome.
 
 ### Prerequisites
-Deploy cluster locally using `skaffold dev -p=only-mock` (configured to poll mocked data)
+Deploy cluster locally using `skaffold dev`
 ### Run tests locally
 * Install all dependencies: `pip install --user -r tests/integration/requirements.txt` 
 * Can be run in Visual Studio Code by opening individual tests and run `Python: Pytest` debug configuration