Feat: improve polling time merge 1 (#1132)

* # Conflicts: # .github/workflows/agreements.yaml # .github/workflows/cd-pages.yaml # .github/workflows/ci-main.yaml # .github/workflows/ci-release-pr.yaml # .github/workflows/ci-release.yaml # .github/workflows/release-notes.yaml # .gitignore # CHANGELOG.md # charts/splunk-connect-for-snmp/Chart.lock # charts/splunk-connect-for-snmp/Chart.yaml # charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl # create_packages.sh # docker_compose/.env # docker_compose/Corefile # docker_compose/manage_secrets.py # docs/bestpractices.md # docs/dockercompose/2-download-package.md # docs/dockercompose/3-inventory-configuration.md # docs/dockercompose/4-scheduler-configuration.md # docs/dockercompose/5-traps-configuration.md # docs/dockercompose/6-env-file-configuration.md # docs/dockercompose/7-snmpv3-secrets.md # docs/gui/enable-gui.md # mkdocs.yml # poetry.lock # pyproject.toml # rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_autoscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml # rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml # rendered/manifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml # splunk_connect_for_snmp/__init__.py # splunk_connect_for_snmp/enrich/tasks.py # splunk_connect_for_snmp/inventory/loader.py # splunk_connect_for_snmp/snmp/auth.py # splunk_connect_for_snmp/snmp/manager.py # splunk_connect_for_snmp/splunk/tasks.py # splunk_connect_for_snmp/traps.py # test/snmp/test_auth.py * chore: update lock * test: test [run-int-tests]
splunk · Nov 26, 2024 · 0aff22e · 0aff22e
1 parent b70254c
commit 0aff22e
Show file tree

Hide file tree

Showing 180 changed files with 7,334 additions and 2,066 deletions.
diff --git a/.github/workflows/agreements.yaml b/.github/workflows/agreements.yaml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-workflow-agreements:
-    uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-agreements.yaml@v1.4.2
+    uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-agreements.yaml@v1.7.0
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/cd-dashboard-release.yaml b/.github/workflows/cd-dashboard-release.yaml
@@ -0,0 +1,22 @@
+name: dashboard-release
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  dashboard-release:
+    name: Dashboard release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      packages: write
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+      - name: Upload dashboard to release assets
+        run: |
+          VERSION=$(echo $GITHUB_REF | cut -d / -f 3)
+          gh release upload $VERSION dashboard/dashboard.xml
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/cd-docs-pdf.yaml b/.github/workflows/cd-docs-pdf.yaml
@@ -38,6 +38,7 @@ jobs:
       - id: doc-files
         name: Produce PDF from docs
         run: |
+          sudo apt-get update
           sudo apt-get install -y pandoc texlive librsvg2-bin texlive-latex-extra
           pandoc -s --pdf-engine=pdflatex -o /tmp/SC4S4SNMP_docs.pdf $(find . -type f -name "*.md") -H deeplists.tex
       - name: Release

diff --git a/.github/workflows/cd-pages.yaml b/.github/workflows/cd-pages.yaml
@@ -29,7 +29,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to GitHub Packages Docker Registry
-        uses: docker/login-action@v3.1.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -52,7 +52,7 @@ jobs:
             type=sha,format=long
       - name: Build and push action
         id: docker_action_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true

diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml
@@ -59,18 +59,18 @@ jobs:
           FOSSA_API_KEY: ${{ secrets.FOSSA_API_KEY }}
   semgrep-scan:
     name: semgrep
-    runs-on: ubuntu-latest
     if: (github.actor != 'dependabot[bot]')
-    steps:
-      - uses: actions/checkout@v4
-      - uses: semgrep/semgrep-action@v1
-        with:
-          publishToken: ${{ secrets.SEMGREP_APP_TOKEN }}
+    uses: splunk/sast-scanning/.github/workflows/sast-scan.yml@main
+    secrets: inherit
+    with:
+      block_mode: "on"
   pre-commit:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
       - uses: pre-commit/[email protected]
   test-unit:
     name: Test Unit Python ${{ matrix.python-version }}
@@ -128,8 +128,8 @@ jobs:
       commit_message:
         echo "${{ steps.get_commit_message.outputs.commit_message }}"
 
-  test-integration:
-    name: Run integration tests
+  test-integration-microk8s:
+    name: Run integration tests in microk8s deployment
     needs:
       - integration-tests-check
     runs-on: ubuntu-latest
@@ -138,17 +138,55 @@ jobs:
     steps:
       - name: Checkout Project
         uses: actions/checkout@v4
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
       - name: run install_microk8s.sh
         run: |
           sudo snap install microk8s --classic --channel=1.30/stable
           sudo apt-get install snmp -y
           sudo apt-get install python3-dev -y
-      - name: run automatic_setup.sh
-        run: integration_tests/automatic_setup.sh integration
+      - name: run automatic_setup_microk8s.sh
+        run: integration_tests/automatic_setup_microk8s.sh integration
       - name: run tests
         working-directory: integration_tests
         run: |
-          poetry run pytest --splunk_host="localhost" --splunk_password="changeme2" --trap_external_ip="$(hostname -I | cut -d " " -f1)"
-
+          poetry run pytest --splunk_host="localhost" --splunk_password="changeme2" --trap_external_ip="$(hostname -I | cut -d " " -f1)" --sc4snmp_deployment="microk8s"
 
+  test-integration-compose:
+    name: Run integration tests in docker compose deployment
+    needs:
+      - integration-tests-check
+    runs-on: ubuntu-latest
+    if: "contains(needs.integration-tests-check.outputs.commit_message, '[run-int-tests]')"
+    timeout-minutes: 120
+    steps:
+      - name: Checkout Project
+        uses: actions/checkout@v4
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - name: Install docker compose
+        run: |
+          # Add Docker's official GPG key:
+          sudo apt-get update
+          sudo apt-get install ca-certificates curl
+          sudo install -m 0755 -d /etc/apt/keyrings
+          sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+          sudo chmod a+r /etc/apt/keyrings/docker.asc
 
+          # Add the repository to Apt sources:
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+            $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+            sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+      - name: run automatic_setup_compose.sh
+        run: integration_tests/automatic_setup_compose.sh integration
+      - name: run tests
+        working-directory: integration_tests
+        run: |
+          poetry run pytest --splunk_host="localhost" --splunk_password="changeme2" --trap_external_ip="$(hostname -I | cut -d " " -f1)" --sc4snmp_deployment="docker-compose"
diff --git a/.github/workflows/ci-release-pr.yaml b/.github/workflows/ci-release-pr.yaml
@@ -42,7 +42,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to GitHub Packages Docker Registry
-        uses: docker/login-action@v3.1.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -62,7 +62,7 @@ jobs:
             type=ref,event=branch
       - name: Build and push action
         id: docker_action_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: false

diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml
@@ -43,7 +43,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to GitHub Packages Docker Registry
-        uses: docker/login-action@v3.1.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -64,7 +64,7 @@ jobs:
             type=ref,event=pr
       - name: Build and push action
         id: docker_action_build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           push: true
@@ -77,7 +77,7 @@ jobs:
           node-version: "14"
       - name: Semantic Release
         id: version
-        uses: splunk/[email protected].4
+        uses: splunk/[email protected].9
         with:
           git_committer_name: ${{ secrets.SA_GH_USER_NAME }}
           git_committer_email: ${{ secrets.SA_GH_USER_EMAIL }}

diff --git a/.github/workflows/ci-ui-tests.yaml b/.github/workflows/ci-ui-tests.yaml
@@ -75,8 +75,8 @@ jobs:
           sudo apt-get install snmp -y
           sudo apt-get install python3-dev -y
 
-      - name: run automatic_setup.sh
-        run: integration_tests/automatic_setup.sh
+      - name: run automatic_setup_microk8s.sh
+        run: integration_tests/automatic_setup_microk8s.sh
 
       - name: install dependencies
         working-directory: ui_tests

diff --git a/.github/workflows/mike.yaml b/.github/workflows/mike.yaml
@@ -45,7 +45,7 @@ jobs:
           sudo apt install gnome-keyring
           BRANCH=$(echo $GITHUB_REF | cut -d / -f 3)
           echo $BRANCH
-          pip3 install poetry=="1.5.1"
+          curl -sSL https://install.python-poetry.org | python3 -
           poetry install
           poetry run pip install 'setuptools==65.6.3'
           poetry run mike deploy -p $BRANCH
diff --git a/.github/workflows/offline-installation.yaml b/.github/workflows/offline-installation.yaml
@@ -30,7 +30,7 @@ jobs:
           PYTHON_SCRIPT=$(pwd)/${{ env.GET_YAML_FIELD_SCRIPT }}
           ./${{ env.CREATE_PACKAGES_SCRIPT }} "$PYTHON_SCRIPT"
           VERSION=$(echo $GITHUB_REF | cut -d / -f 3)
-          gh release upload $VERSION /tmp/package/packages/*
+          gh release upload --clobber $VERSION /tmp/package/packages/*
         env:
           CREATE_PACKAGES_SCRIPT: create_packages.sh
           GET_YAML_FIELD_SCRIPT: get_yaml_fields.py

diff --git a/.github/workflows/release-notes.yaml b/.github/workflows/release-notes.yaml
@@ -10,6 +10,6 @@ jobs:
     permissions:
       contents: write
       packages: write
-    uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-release-notes.yaml@v1.4.2
+    uses: splunk/addonfactory-github-workflows/.github/workflows/reusable-release-notes.yaml@v1.7.0
     secrets:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -104,6 +104,7 @@ celerybeat.pid
 # Environments
 .env
 !docker_compose/.env
+!integration_tests/.env
 .venv
 env/
 venv/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,27 +15,28 @@
 #
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.6.0
     hooks:
     -   id: check-merge-conflict
     -   id: debug-statements
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+    rev: v3.17.0
     hooks:
     -   id: pyupgrade
         args:
         - --py3-plus
 -   repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.8.0
     hooks:
     -   id: black
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
     -   id: isort
         args: ["--profile", "black"]
 -    repo: https://github.com/pre-commit/mirrors-mypy
-     rev: v0.931
+     rev: v1.11.2
      hooks:
      -  id: mypy
-        additional_dependencies: [types-all]
+        exclude: (^ui_tests|^test*|^docs)
+        additional_dependencies: [types-requests, types-PyYAML]
diff --git a/.semgrepignore b/.semgrepignore
@@ -1,2 +1,6 @@
-Dockerfile
-integration_tests/
+docs/
+test/
+ui_tests/
+integration_tests/
+rendered/
+examples/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,13 +2,44 @@
 
 ## Unreleased
 
+### Changed
+- add IF-MIB.ifOperStatus to baseIF profile
+- allow to set subnets for docker network configuration from .env file
+
+### Fixed
+
+## [1.12.1]
+
+### Changed
+- general refactor of documentation
+- merge docker compose files into one
+
+### Fixed
+- dns resolution for ipv6
+
+## [1.12.0]
+
+### Changed
+- add metrics dashboard
+- add support for IPv6 polling and traps for kubernetes deployment
+- add support for IPv6 polling and traps for docker deployment
+- add nodeSelector and podAntiAffinity to sim chart
+- add support for new IP format in dashboard
+
+## [1.11.0]
+
 ### Changed
 - add docker compose deployment
 - update default microk8s to 1.30
 - change group and user ID in Docker image to 10001:10001
 - create and/or change ownership of /tmp and /.pysnmp to 10001:10001 in Dockerfile
 - while mounting volumes in docker compose explicitly mount them as read only or read write
-- change nonReapeater to 0 and add maxRepetitions as configurable
+- added `values.yaml` schema validation using `values.schema.json`
+- released beta version of improved polling performance
+- added `yamllint` validation for the `values.yaml` formatting
+- added "in code" validation of groups and profiles
+- added logs configuration to docker compose deployment
+- add support for different security level in snmp v3
 
 ### Fixed
 - fixed a bug with configuration from values.yaml not being transferred to the UI while migrating to SC4SNMP-UI

diff --git a/charts/splunk-connect-for-snmp/Chart.lock b/charts/splunk-connect-for-snmp/Chart.lock
@@ -4,9 +4,9 @@ dependencies:
   version: 13.18.5
 - name: redis
   repository: https://charts.bitnami.com/bitnami
-  version: 19.1.0
+  version: 20.2.0
 - name: mibserver
   repository: https://pysnmp.github.io/mibs/charts/
-  version: 1.15.8
-digest: sha256:25bb231b140ccc7aed166d3a142ccfa81e5026560d5009f8b0dedde9ecb0657a
-generated: "2024-04-17T09:12:38.843604607Z"
+  version: 1.15.13
+digest: sha256:dedfae30b39e9b78a83f8e1822666095f79ba2b9b112b04f7a11a2a318674713
+generated: "2024-10-14T10:28:52.408747265Z"
diff --git a/charts/splunk-connect-for-snmp/Chart.yaml b/charts/splunk-connect-for-snmp/Chart.yaml
@@ -14,19 +14,19 @@ type: application
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
-version: 1.11.0-beta.8
+version: 1.12.1-beta.7
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.11.0-beta.8"
+appVersion: "1.12.1-beta.7"
 #
 dependencies:
   - name: mongodb
     version: ~13.18.0
     repository: https://charts.bitnami.com/bitnami
   - name: redis
-    version: ~19.1.0
+    version: ~20.2.0
     repository: https://charts.bitnami.com/bitnami
   - name: mibserver
     version: ~1.15

diff --git a/charts/splunk-connect-for-snmp/templates/common/scheduler-config.yaml b/charts/splunk-connect-for-snmp/templates/common/scheduler-config.yaml
@@ -6,10 +6,6 @@ metadata:
     {{- include "splunk-connect-for-snmp.scheduler.labels" . | nindent 4 }}
 data:
   config.yaml: |-
-{{- if (.Values.scheduler).usernames }}
-      usernames:
-{{ .Values.scheduler.usernames | indent 8 -}}
-{{ end }}
 {{- if (.Values.scheduler).profiles }}
       profiles:
 {{ .Values.scheduler.profiles | indent 8 -}}

diff --git a/charts/splunk-connect-for-snmp/templates/scheduler/deployment.yaml b/charts/splunk-connect-for-snmp/templates/scheduler/deployment.yaml
@@ -59,8 +59,6 @@ spec:
               value: "http://{{ printf "%s-%s" .Release.Name "mibserver" }}/standard.txt"
             - name: LOG_LEVEL
               value: {{ .Values.scheduler.logLevel | default "INFO" }}
-            - name: INVENTORY_REFRESH_RATE
-              value: {{ .Values.scheduler.inventory_refresh_rate | default "600" | quote }}
           volumeMounts:
             - name: config
               mountPath: "/app/config"