feat: add processors and receivers for K8S telemetry proxy (#4178)

Co-authored-by: Guy Moses <[email protected]>
lumigo-io · Feb 17, 2023 · fbfc5cb · fbfc5cb
1 parent 381e67b
commit fbfc5cb
Show file tree

Hide file tree

Showing 25 changed files with 189 additions and 1,315 deletions.
diff --git a/.github/workflows/build-and-test-windows.yml b/.github/workflows/build-and-test-windows.yml
@@ -2,9 +2,9 @@ name: build-and-test-windows
 on:
   push:
     branches: [ main ]
-    tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+*'
-  pull_request:
+  #   tags:
+  #     - 'v[0-9]+.[0-9]+.[0-9]+*'
+  # pull_request:
 env:
   TEST_RESULTS: testbed/tests/results/junit/results.xml
   # See: https://github.com/actions/cache/issues/810#issuecomment-1222550359

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -1,7 +1,7 @@
 name: build-and-test
 on:
   push:
-    branches: [ main ]
+    branches: [ lumigo-main ]
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+*'
   pull_request:
@@ -49,16 +49,16 @@ jobs:
         uses: actions/checkout@v2
       - name: Check Collector Module Version
         run: ./.github/workflows/scripts/check-collector-module-version.sh
-  check-codeowners:
-    runs-on: ubuntu-latest
-    needs: [setup-environment]
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v2
-      - name: Check Code Owner Existence
-        run: ./.github/workflows/scripts/check-codeowners.sh check_code_owner_existence
-      - name: Check Component Existence
-        run: ./.github/workflows/scripts/check-codeowners.sh check_component_existence
+  # check-codeowners:
+  #   runs-on: ubuntu-latest
+  #   needs: [setup-environment]
+  #   steps:
+  #     - name: Checkout Repo
+  #       uses: actions/checkout@v2
+  #     - name: Check Code Owner Existence
+  #       run: ./.github/workflows/scripts/check-codeowners.sh check_code_owner_existence
+  #     - name: Check Component Existence
+  #       run: ./.github/workflows/scripts/check-codeowners.sh check_component_existence
   lint-matrix:
     strategy:
       matrix:
@@ -312,26 +312,26 @@ jobs:
         os:
           - darwin
           - linux
-          - windows
+          # - windows
         arch:
-          - 386
+          # - 386
           - amd64
-          - arm
+          # - arm
           - arm64
-          - ppc64le
-        exclude:
-          - os: darwin
-            arch: 386
-          - os: darwin
-            arch: arm
-          - os: darwin
-            arch: ppc64le
-          - os: windows
-            arch: arm
-          - os: windows
-            arch: arm64
-          - os: windows
-            arch: ppc64le
+          # - ppc64le
+        # exclude:
+        #   - os: darwin
+        #     arch: 386
+        #   - os: darwin
+        #     arch: arm
+        #   - os: darwin
+        #     arch: ppc64le
+        #   - os: windows
+        #     arch: arm
+        #   - os: windows
+        #     arch: arm64
+        #   - os: windows
+        #     arch: ppc64le
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
@@ -362,6 +362,7 @@ jobs:
           path: ./bin/*
 
   build-package:
+    if: github.repository == 'open-telemetry/opentelemetry-collector-contrib' # skip, not needed for the Lumigo builds
     runs-on: ubuntu-latest
     needs: [cross-compile]
     strategy:
@@ -444,6 +445,7 @@ jobs:
           path: ./dist/*.msi
 
   publish-check:
+    if: github.repository == 'open-telemetry/opentelemetry-collector-contrib' # skip, not needed for the Lumigo builds
     runs-on: ubuntu-latest
     needs: [build-package]
     steps:
@@ -462,9 +464,9 @@ jobs:
         id: check
         run: ./.github/workflows/scripts/verify-dist-files-exist.sh
   publish-dev:
+    if: github.repository == 'open-telemetry/opentelemetry-collector-contrib' # skip, not needed for the Lumigo builds
     runs-on: ubuntu-latest
     needs: [lint, unittest, integration-tests, build-package]
-    if: (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) && github.repository == 'open-telemetry/opentelemetry-collector-contrib'
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
@@ -526,20 +528,83 @@ jobs:
         run: |
             docker push otel/opentelemetry-collector-contrib-dev:$GITHUB_SHA
             docker push otel/opentelemetry-collector-contrib-dev:latest
-  publish-stable:
+
+  publish-draft:
     runs-on: ubuntu-latest
-    needs: [lint, unittest, integration-tests, build-package]
-    if: startsWith(github.ref, 'refs/tags/v') && github.repository == 'open-telemetry/opentelemetry-collector-contrib'
+    permissions: write-all
+    needs:
+    - lint
+    - unittest
+    - integration-tests
+    # We depend on cross-compile instead of build-package to make sure we get all the binaries we need
+    - cross-compile
+    # - build-package # skip, not used for the Lumigo builds
+    # We build this for PRs (their reference looks likme `refs/pull/<pr_id>/head` )
+    if: startsWith(github.ref, 'refs/pull/') && github.repository == 'lumigo-io/opentelemetry-collector-contrib'
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v3
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
           go-version: 1.19
+      - name: Cache Go
+        id: go-cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/bin
+            ~/go/pkg/mod
+          key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
+      - name: Install Tools
+        if: steps.go-cache.outputs.cache-hit != 'true'
+        run: make install-tools
       - name: Mkdir bin and dist
         run: |
           mkdir bin/ dist/
+      - name: Download Binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: collector-binaries
+          path: ./bin/
+      - name: Verify Distribution Files Exist
+        id: check
+        run: ./.github/workflows/scripts/verify-release-artifacts-exist.sh
+      - name: Set Pre-Release Tag
+        id: github_tag
+        run: ./.github/workflows/scripts/set_prerelease_tag.sh
+      - name: Set PR Identifier
+        id: github_pr_id
+        run: ./.github/workflows/scripts/set_pr_id.sh
+      - uses: rickstaa/action-create-tag@v1
+        with:
+          tag: ${{ steps.github_tag.outputs.tag }}
+          message: "PR ${{ steps.github_pr_id.outputs.id }} draft-release"
+      - name: Create Github Release
+        run: |
+          ghr -n "${RELEASE_TAG}" -prerelease -t "${GITHUB_TOKEN}" -u "${GITHUB_REPOSITORY_OWNER}" -r opentelemetry-collector-contrib -replace "${RELEASE_TAG}" bin/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          RELEASE_TAG: ${{ steps.github_tag.outputs.tag }}
+
+  publish-stable:
+    runs-on: ubuntu-latest
+    needs:
+    - lint
+    - unittest
+    - integration-tests
+    # We depend on cross-compile instead of build-package to make sure we get all the binaries we need
+    - cross-compile
+    # - build-package # skip, not used for the Lumigo builds
+    # Note that the tags for pre-releases start with 'pr', so it does not trigger this job
+    if: startsWith(github.ref, 'refs/tags/v') && endsWith(github.ref, '-lumigo') && github.repository == 'lumigo-io/opentelemetry-collector-contrib'
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
       - name: Cache Go
         id: go-cache
         uses: actions/cache@v3
@@ -548,19 +613,26 @@ jobs:
             ~/go/bin
             ~/go/pkg/mod
           key: go-cache-${{ runner.os }}-${{ hashFiles('**/go.sum') }}
-      - name: Install dependencies
-        if: steps.go-cache.outputs.cache-hit != 'true'
-        run: make -j2 gomoddownload
       - name: Install Tools
         if: steps.go-cache.outputs.cache-hit != 'true'
         run: make install-tools
+      - name: Mkdir bin and dist
+        run: |
+          mkdir bin/ dist/
+      - name: Download Binaries
+        uses: actions/download-artifact@v3
+        with:
+          name: collector-binaries
+          path: ./bin/
+      - name: Verify Distribution Files Exist
+        id: check
+        run: ./.github/workflows/scripts/verify-release-artifacts-exist.sh
       - name: Set Release Tag
         id: github_tag
         run: ./.github/workflows/scripts/set_release_tag.sh
       - name: Create Github Release
         run: |
-          mkdir -p dist/
-          ghr -t $GITHUB_TOKEN -u open-telemetry -r opentelemetry-collector-contrib --replace $RELEASE_TAG dist/
+          ghr -t $GITHUB_TOKEN -u $GITHUB_REPOSITORY_OWNER -r opentelemetry-collector-contrib --replace $RELEASE_TAG bin/
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           RELEASE_TAG: ${{ steps.github_tag.outputs.tag }}

diff --git a/.github/workflows/scripts/set_pr_id.sh b/.github/workflows/scripts/set_pr_id.sh
@@ -0,0 +1,4 @@
+# The GitHub Ref is going to be a Pull ref, e.g., `refs/pull/123/head`
+PR_ID="$(echo \"${GITHUB_REF}\" | sed -e 's|refs/pull/\([0-9]*\)/.*|\1|')"
+
+echo "id=$PR_ID" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/set_prerelease_tag.sh b/.github/workflows/scripts/set_prerelease_tag.sh
@@ -0,0 +1,4 @@
+# The GitHub Ref is going to be a Pull ref, e.g., `refs/pull/123/head`
+TAG="pr-$(echo \"${GITHUB_REF}\" | awk -F/ '{ print $3 }')-${GITHUB_SHA:0:7}-lumigo"
+
+echo "tag=$TAG" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/set_release_tag.sh b/.github/workflows/scripts/set_release_tag.sh
@@ -1,5 +1,5 @@
 TAG="${GITHUB_REF##*/}"
 if [[ $TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+.* ]]
 then
-    echo "::set-output name=tag::$TAG"
+    echo "tag=$TAG" >> $GITHUB_OUTPUT
 fi
diff --git a/.github/workflows/scripts/setup_e2e_tests.sh b/.github/workflows/scripts/setup_e2e_tests.sh
@@ -15,4 +15,4 @@ else
 fi
 done
 MATRIX+=",{\"test\":\"$curr\"}]}"
-echo "::set-output name=loadtest_matrix::$MATRIX"
+echo "loadtest_matrix=$MATRIX" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/setup_stability_tests.sh b/.github/workflows/scripts/setup_stability_tests.sh
@@ -8,4 +8,4 @@ for i in "${!TESTS[@]}"; do
     MATRIX+="{\"test\":\"$curr\"},"
 done
 MATRIX+="]}"
-echo "::set-output name=stabilitytest_matrix::$MATRIX"
+echo "stabilitytest_matrix=$MATRIX" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/verify-dist-files-exist.sh b/.github/workflows/scripts/verify-dist-files-exist.sh
@@ -19,8 +19,8 @@ do
     if [[ ! -f $f ]]
     then
         echo "$f does not exist."
-        echo "::set-output name=passed::false"
+        echo "passed=false" >> $GITHUB_OUTPUT
         exit 0
     fi
 done
-echo "::set-output name=passed::true"
+echo "passed=true" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/scripts/verify-release-artifacts-exist.sh b/.github/workflows/scripts/verify-release-artifacts-exist.sh
@@ -0,0 +1,16 @@
+files=(
+    bin/otelcontribcol_darwin_arm64
+    bin/otelcontribcol_darwin_amd64
+    bin/otelcontribcol_linux_arm64
+    bin/otelcontribcol_linux_amd64
+);
+for f in "${files[@]}"
+do
+    if [[ ! -f $f ]]
+    then
+        echo "$f does not exist."
+        echo "passed=false" >> $GITHUB_OUTPUT
+        exit 0
+    fi
+done
+echo "passed=true" >> $GITHUB_OUTPUT
diff --git a/builder/builder_configuration.yaml b/builder/builder_configuration.yaml
@@ -4,23 +4,24 @@ exporters:
     gomod: "go.opentelemetry.io/collector v0.60.0"
 
 extensions:
+  - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/extension/headers_setter v0.60.0"
   - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.60.0"
   - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/extension/lumigoauthextension v0.60.0"
 
 receivers:
   - import: go.opentelemetry.io/collector/receiver/otlpreceiver
     gomod: "go.opentelemetry.io/collector v0.60.0"
+  - import: "go.opentelemetry.io/collector/receiver/k8seventsreceiver v0.60.0"
 
 processors:
   - import: go.opentelemetry.io/collector/processor/batchprocessor
     gomod: go.opentelemetry.io/collector v0.60.0
   - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/processor/filterprocessor v0.60.0"
-  - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionbykeyprocessor v0.60.0"
+  - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/processor/k8sattributesprocessor v0.60.0"
   - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.60.0"
   - gomod: "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.60.0"
 
 replaces:
   # a list of "replaces" directives that will be part of the resulting go.mod
   - github.com/open-telemetry/opentelemetry-collector-contrib/extension/lumigoauthextension v0.60.0 => github.com/lumigo-io/opentelemetry-collector-contrib/extension/lumigoauthextension 585478a75f3c3c7fe280e98b7aa43b1ad0dcf344
   - github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionbykeyprocessor v0.60.0 => github.com/lumigo-io/opentelemetry-collector-contrib/processor/redactionbykeyprocessor f4bad13377e0d998520063f09759b932d6806972
-#  - github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcehasherprocessor v0.60.0 => github.com/lumigo-io/opentelemetry-collector-contrib/processor/resourcehasherprocessor 1306ffa7a6f605571f7f265f2e71e8b0957588da
diff --git a/cmd/configschema/go.mod b/cmd/configschema/go.mod
@@ -326,6 +326,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/headerssetter v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/httpforwarder v0.60.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/extension/lumigoauthextension v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/ecstaskobserver v0.60.0 // indirect
@@ -374,6 +375,7 @@ require (
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricsgenerationprocessor v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/metricstransformprocessor v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor v0.60.0 // indirect
+	github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionbykeyprocessor v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor v0.60.0 // indirect
 	github.com/open-telemetry/opentelemetry-collector-contrib/processor/routingprocessor v0.60.0 // indirect
@@ -725,10 +727,12 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/heal
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/httpforwarder => ../../extension/httpforwarder
 
-replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension => ../../extension/oauth2clientauthextension
+replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/lumigoauthextension => ../../extension/lumigoauthextension
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer => ../../extension/observer
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/oauth2clientauthextension => ../../extension/oauth2clientauthextension
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/ecstaskobserver => ../../extension/observer/ecstaskobserver
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/extension/observer/hostobserver => ../../extension/observer/hostobserver
@@ -789,6 +793,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/metr
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/probabilisticsamplerprocessor => ../../processor/probabilisticsamplerprocessor/
 
+replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionbykeyprocessor => ../../processor/redactionbykeyprocessor/
+
 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourcedetectionprocessor => ../../processor/resourcedetectionprocessor/
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/resourceprocessor => ../../processor/resourceprocessor/

diff --git a/cmd/telemetrygen/internal/telemetrygen/config.go b/cmd/telemetrygen/internal/telemetrygen/config.go
@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package telemetrygen // import "github.com/open-telemetry/opentelemetry-collector-contrib/telemetrygen/internal/telemetrygen"
+package telemetrygen // import "github.com/open-telemetry/opentelemetry-collector-contrib/cmd/telemetrygen/internal/telemetrygen"
 
 import (
 	"errors"

diff --git a/extension/lumigoauthextension/config.go b/extension/lumigoauthextension/config.go
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package lumigoauthextension // import "github.com/lumigo-io/opentelemetry-collector-contrib/extension/lumigoauthextension"
+package lumigoauthextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/lumigoauthextension"
 
 import (
 	"go.opentelemetry.io/collector/config"