diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a82f404..7abd495ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## Unreleased ### Changed +- general refactor of documentation +- merge docker compose files into one ### Fixed diff --git a/docker_compose/docker-compose-coredns.yaml b/docker_compose/docker-compose-coredns.yaml deleted file mode 100644 index dcabb1496..000000000 --- a/docker_compose/docker-compose-coredns.yaml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3.8' -services: - coredns: - image: ${COREDNS_IMAGE}:${COREDNS_TAG:-latest} - command: ["-conf", "/Corefile"] - container_name: coredns - restart: on-failure - expose: - - '53' - - '53/udp' - volumes: - - '${COREFILE_ABS_PATH}:/Corefile' - networks: - sc4snmp_network: - ipv4_address: ${COREDNS_ADDRESS} diff --git a/docker_compose/docker-compose-dependencies.yaml b/docker_compose/docker-compose-dependencies.yaml deleted file mode 100644 index 73434af1c..000000000 --- a/docker_compose/docker-compose-dependencies.yaml +++ /dev/null @@ -1,40 +0,0 @@ -version: '3.8' -services: - snmp-mibserver: - image: ${MIBSERVER_IMAGE}:${MIBSERVER_TAG:-latest} - container_name: snmp-mibserver - environment: - - NGINX_ENTRYPOINT_QUIET_LOGS=${NGINX_ENTRYPOINT_QUIET_LOGS:-1} - volumes: - - snmp-mibserver-tmp:/tmp/ - depends_on: - - coredns - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} - - redis: - image: ${REDIS_IMAGE}:${REDIS_TAG:-latest} - container_name: redis - restart: always - environment: - - ALLOW_EMPTY_PASSWORD=yes - depends_on: - - coredns - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} - mongo: - image: ${MONGO_IMAGE}:${MONGO_TAG:-latest} - container_name: mongo - restart: always - depends_on: - - coredns - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} -volumes: - snmp-mibserver-tmp: null diff --git a/docker_compose/docker-compose-inventory.yaml b/docker_compose/docker-compose-inventory.yaml deleted file mode 100644 index 91cee276c..000000000 --- a/docker_compose/docker-compose-inventory.yaml +++ /dev/null @@ -1,36 +0,0 @@ -version: '3.8' -services: - inventory: - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - container_name: sc4snmp-inventory - command: ["inventory"] - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - - # Inventory configuration - - LOG_LEVEL=${INVENTORY_LOG_LEVEL:-INFO} - - CHAIN_OF_TASKS_EXPIRY_TIME=${CHAIN_OF_TASKS_EXPIRY_TIME:-500} - - CONFIG_FROM_MONGO=${CONFIG_FROM_MONGO:-false} - depends_on: - - redis - - mongo - - coredns - volumes: - - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - ${INVENTORY_FILE_ABSOLUTE_PATH}:/app/inventory/inventory.csv:ro - - inventory-pysnmp-cache-volume:/.pysnmp/:rw - - inventory-tmp:/tmp/:rw - restart: on-failure - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} -volumes: - inventory-tmp: null - inventory-pysnmp-cache-volume: null diff --git a/docker_compose/docker-compose-network.yaml b/docker_compose/docker-compose-network.yaml deleted file mode 100644 index f7fa80a7e..000000000 --- a/docker_compose/docker-compose-network.yaml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3.8' -networks: - sc4snmp_network: - name: sc4snmp_network - enable_ipv6: ${IPv6_ENABLED:-false} - ipam: - config: - - subnet: 172.28.0.0/16 - gateway: 172.28.0.1 - - subnet: fd02::/64 - gateway: fd02::1 \ No newline at end of file diff --git a/docker_compose/docker-compose-scheduler.yaml b/docker_compose/docker-compose-scheduler.yaml deleted file mode 100644 index f74c1e072..000000000 --- a/docker_compose/docker-compose-scheduler.yaml +++ /dev/null @@ -1,33 +0,0 @@ -version: '3.8' -services: - scheduler: - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - container_name: sc4snmp-scheduler - command: ["celery", "beat"] - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - - # Scheduler configuration - - LOG_LEVEL=${SCHEDULER_LOG_LEVEL:-INFO} - depends_on: - - redis - - mongo - - coredns - volumes: - - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - scheduler-pysnmp-cache-volume:/.pysnmp/:rw - - scheduler-tmp:/tmp/:rw - restart: on-failure - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} -volumes: - scheduler-tmp: null - scheduler-pysnmp-cache-volume: null \ No newline at end of file diff --git a/docker_compose/docker-compose-secrets.yaml b/docker_compose/docker-compose-secrets.yaml deleted file mode 100644 index c1dae5281..000000000 --- a/docker_compose/docker-compose-secrets.yaml +++ /dev/null @@ -1,2 +0,0 @@ -secrets: {} -version: '3.8' diff --git a/docker_compose/docker-compose-traps.yaml b/docker_compose/docker-compose-traps.yaml deleted file mode 100644 index 1abba1b02..000000000 --- a/docker_compose/docker-compose-traps.yaml +++ /dev/null @@ -1,51 +0,0 @@ -services: - traps: - command: - - trap - container_name: sc4snmp-traps - depends_on: - - redis - - mongo - - coredns - dns: - - ${COREDNS_ADDRESS} - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - - LOG_LEVEL=${TRAP_LOG_LEVEL:-INFO} - - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} - - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} - - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} - - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} - - SNMP_V3_SECURITY_ENGINE_ID=${SNMP_V3_SECURITY_ENGINE_ID:-80003a8c04} - - PYSNMP_DEBUG=${PYSNMP_DEBUG} - - IPv6_ENABLED=${IPv6_ENABLED:-false} - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - networks: - - sc4snmp_network - ports: - - mode: host - protocol: udp - published: ${TRAPS_PORT} - target: 2162 - - mode: host - protocol: udp - published: ${IPv6_TRAPS_PORT} - target: 2163 - restart: on-failure - secrets: [] - volumes: - - ${TRAPS_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - traps-pysnmp-cache-volume:/.pysnmp/:rw - - traps-tmp:/tmp/:rw -version: '3.8' -volumes: - traps-pysnmp-cache-volume: null - traps-tmp: null diff --git a/docker_compose/docker-compose-worker-poller.yaml b/docker_compose/docker-compose-worker-poller.yaml deleted file mode 100644 index 8f52118cf..000000000 --- a/docker_compose/docker-compose-worker-poller.yaml +++ /dev/null @@ -1,69 +0,0 @@ -services: - worker-poller: - command: - - celery - - worker-poller - depends_on: - - redis - - mongo - - coredns - dns: - - ${COREDNS_ADDRESS} - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} - - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} - - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} - - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} - - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} - - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} - - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} - - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} - - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} - - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} - - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} - - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} - - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} - - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} - - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} - - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} - - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} - - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} - - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} - - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} - - WORKER_CONCURRENCY=${WORKER_POLLER_CONCURRENCY:-2} - - PREFETCH_COUNT=${PREFETCH_POLLER_COUNT:-1} - - PYSNMP_DEBUG=${PYSNMP_DEBUG} - - IPv6_ENABLED=${IPv6_ENABLED:-false} - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - networks: - - sc4snmp_network - restart: on-failure - secrets: [] - volumes: - - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - worker-poller-pysnmp-cache-volume:/.pysnmp/:rw - - worker-poller-tmp:/tmp/:rw - deploy: - mode: replicated - replicas: ${WORKER_POLLER_REPLICAS:-2} - resources: - limits: - cpus: ${WORKER_POLLER_CPU_LIMIT:-0.50} - memory: ${WORKER_POLLER_MEMORY_LIMIT:-500M} - reservations: - cpus: ${WORKER_POLLER_CPU_RESERVATIONS:-0.25} - memory: ${WORKER_POLLER_MEMORY_RESERVATIONS:-250M} -version: '3.8' -volumes: - worker-poller-pysnmp-cache-volume: null - worker-poller-tmp: null diff --git a/docker_compose/docker-compose-worker-sender.yaml b/docker_compose/docker-compose-worker-sender.yaml deleted file mode 100644 index f0040e6ae..000000000 --- a/docker_compose/docker-compose-worker-sender.yaml +++ /dev/null @@ -1,70 +0,0 @@ -version: '3.8' -services: - worker-sender: - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - command: ["celery", "worker-sender"] - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - #- OTEL_METRICS_URL= #If sim enabled - - # Splunk instance configuration - - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} - - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} - - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} - - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} - - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} - - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} - - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} - - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} - - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} - - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} - - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} - - # Workers configuration - - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} - - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} - - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} - - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} - - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} - - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} - - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} - - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} - - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} - - WORKER_CONCURRENCY=${WORKER_SENDER_CONCURRENCY:-2} - - PREFETCH_COUNT=${PREFETCH_SENDER_COUNT:-1} - - PYSNMP_DEBUG=${PYSNMP_DEBUG} - depends_on: - - redis - - mongo - - coredns - volumes: - - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - worker-sender-pysnmp-cache-volume:/.pysnmp/:rw - - worker-sender-tmp:/tmp/:rw - restart: on-failure - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} - deploy: - mode: replicated - replicas: ${WORKER_SENDER_REPLICAS:-1} - resources: - limits: - cpus: ${WORKER_SENDER_CPU_LIMIT:-0.50} - memory: ${WORKER_SENDER_MEMORY_LIMIT:-500M} - reservations: - cpus: ${WORKER_SENDER_CPU_RESERVATIONS:-0.25} - memory: ${WORKER_SENDER_MEMORY_RESERVATIONS:-250M} -volumes: - worker-sender-tmp: null - worker-sender-pysnmp-cache-volume: null \ No newline at end of file diff --git a/docker_compose/docker-compose-worker-trap.yaml b/docker_compose/docker-compose-worker-trap.yaml deleted file mode 100644 index 89f61e54f..000000000 --- a/docker_compose/docker-compose-worker-trap.yaml +++ /dev/null @@ -1,74 +0,0 @@ -version: '3.8' -services: - worker-trap: - image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} - command: ["celery", "worker-trap"] - environment: - - CONFIG_PATH=/app/config/config.yaml - - REDIS_URL=redis://redis:6379/1 - - CELERY_BROKER_URL=redis://redis:6379/0 - - MONGO_URI=mongodb://mongo:27017/ - - SC4SNMP_VERSION=${SC4SNMP_VERSION:-0.0.0} - - MIB_SOURCES=http://snmp-mibserver:8000/asn1/@mib@ - - MIB_INDEX=http://snmp-mibserver:8000/index.csv - - MIB_STANDARD=http://snmp-mibserver:8000/standard.txt - #- OTEL_METRICS_URL= #If sim enabled - - # Splunk instance configuration - - SPLUNK_HEC_HOST=${SPLUNK_HEC_HOST} - - SPLUNK_HEC_SCHEME=${SPLUNK_HEC_PROTOCOL:-https} - - SPLUNK_HEC_PORT=${SPLUNK_HEC_PORT} - - SPLUNK_HEC_TOKEN=${SPLUNK_HEC_TOKEN} - - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false} - - SPLUNK_SOURCETYPE_TRAPS=${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} - - SPLUNK_SOURCETYPE_POLLING_EVENTS=${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} - - SPLUNK_SOURCETYPE_POLLING_METRICS=${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} - - SPLUNK_HEC_INDEX_EVENTS=${SPLUNK_HEC_INDEX_EVENTS:-netops} - - SPLUNK_HEC_INDEX_METRICS=${SPLUNK_HEC_INDEX_METRICS:-netmetrics} - - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector} - - SPLUNK_AGGREGATE_TRAPS_EVENTS=${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} - - IGNORE_EMPTY_VARBINDS=${IGNORE_EMPTY_VARBINDS:-false} - - # Workers configuration - - WALK_RETRY_MAX_INTERVAL=${WALK_RETRY_MAX_INTERVAL:-180} - - WALK_MAX_RETRIES=${WALK_MAX_RETRIES:-5} - - METRICS_INDEXING_ENABLED=${METRICS_INDEXING_ENABLED:-false} - - POLL_BASE_PROFILES=${POLL_BASE_PROFILES:-true} - - IGNORE_NOT_INCREASING_OIDS=${IGNORE_NOT_INCREASING_OIDS:-} - - LOG_LEVEL=${WORKER_LOG_LEVEL:-INFO} - - UDP_CONNECTION_TIMEOUT=${UDP_CONNECTION_TIMEOUT:-3} - - MAX_OID_TO_PROCESS=${MAX_OID_TO_PROCESS:-70} - - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60} - - WORKER_CONCURRENCY=${WORKER_TRAP_CONCURRENCY:-2} - - PREFETCH_COUNT=${PREFETCH_TRAP_COUNT:-1} - - RESOLVE_TRAP_ADDRESS=${RESOLVE_TRAP_ADDRESS:-false} - - MAX_DNS_CACHE_SIZE_TRAPS=${MAX_DNS_CACHE_SIZE_TRAPS:-500} - - TTL_DNS_CACHE_TRAPS=${TTL_DNS_CACHE_TRAPS:-1800} - - PYSNMP_DEBUG=${PYSNMP_DEBUG} - - IPv6_ENABLED=${IPv6_ENABLED:-false} - depends_on: - - redis - - mongo - - coredns - volumes: - - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro - - worker-trap-pysnmp-cache-volume:/.pysnmp/:rw - - worker-trap-tmp:/tmp/:rw - restart: on-failure - networks: - - sc4snmp_network - dns: - - ${COREDNS_ADDRESS} - deploy: - mode: replicated - replicas: ${WORKER_TRAP_REPLICAS:-2} - resources: - limits: - cpus: ${WORKER_TRAP_CPU_LIMIT:-0.50} - memory: ${WORKER_TRAP_MEMORY_LIMIT:-500M} - reservations: - cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} - memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} -volumes: - worker-trap-tmp: null - worker-trap-pysnmp-cache-volume: null \ No newline at end of file diff --git a/docker_compose/docker-compose.yaml b/docker_compose/docker-compose.yaml new file mode 100644 index 000000000..7fd14e719 --- /dev/null +++ b/docker_compose/docker-compose.yaml @@ -0,0 +1,248 @@ +x-general_sc4snmp_data: &general_sc4snmp_data + CONFIG_PATH: /app/config/config.yaml + REDIS_URL: redis://redis:6379/1 + CELERY_BROKER_URL: redis://redis:6379/0 + MONGO_URI: mongodb://mongo:27017/ + MIB_SOURCES: http://snmp-mibserver:8000/asn1/@mib@ + MIB_INDEX: http://snmp-mibserver:8000/index.csv + MIB_STANDARD: http://snmp-mibserver:8000/standard.txt + +x-splunk_general_setup: &splunk_general_setup + SPLUNK_HEC_HOST: ${SPLUNK_HEC_HOST} + SPLUNK_HEC_SCHEME: ${SPLUNK_HEC_PROTOCOL:-https} + SPLUNK_HEC_PORT: ${SPLUNK_HEC_PORT} + SPLUNK_HEC_TOKEN: ${SPLUNK_HEC_TOKEN} + SPLUNK_HEC_INSECURESSL: ${SPLUNK_HEC_INSECURESSL:-false} + SPLUNK_HEC_PATH: ${SPLUNK_HEC_PATH:-/services/collector} + +x-splunk_extended_setup: &splunk_extended_setup + SPLUNK_SOURCETYPE_TRAPS: ${SPLUNK_SOURCETYPE_TRAPS:-sc4snmp:traps} + SPLUNK_SOURCETYPE_POLLING_EVENTS: ${SPLUNK_SOURCETYPE_POLLING_EVENTS:-sc4snmp:event} + SPLUNK_SOURCETYPE_POLLING_METRICS: ${SPLUNK_SOURCETYPE_POLLING_METRICS:-sc4snmp:metric} + SPLUNK_HEC_INDEX_EVENTS: ${SPLUNK_HEC_INDEX_EVENTS:-netops} + SPLUNK_HEC_INDEX_METRICS: ${SPLUNK_HEC_INDEX_METRICS:-netmetrics} + SPLUNK_AGGREGATE_TRAPS_EVENTS: ${SPLUNK_AGGREGATE_TRAPS_EVENTS:-false} + +x-workers_general_setup: &workers_general_setup + SC4SNMP_VERSION: ${SC4SNMP_VERSION:-latest} + IGNORE_EMPTY_VARBINDS: ${IGNORE_EMPTY_VARBINDS:-false} + WALK_RETRY_MAX_INTERVAL: ${WALK_RETRY_MAX_INTERVAL:-180} + WALK_MAX_RETRIES: ${WALK_MAX_RETRIES:-5} + METRICS_INDEXING_ENABLED: ${METRICS_INDEXING_ENABLED:-false} + POLL_BASE_PROFILES: ${POLL_BASE_PROFILES:-true} + IGNORE_NOT_INCREASING_OIDS: ${IGNORE_NOT_INCREASING_OIDS:-} + LOG_LEVEL: ${WORKER_LOG_LEVEL:-INFO} + UDP_CONNECTION_TIMEOUT: ${UDP_CONNECTION_TIMEOUT:-3} + MAX_OID_TO_PROCESS: ${MAX_OID_TO_PROCESS:-70} + PROFILES_RELOAD_DELAY: ${PROFILES_RELOAD_DELAY:-60} + +x-ipv6: &ipv6 + IPv6_ENABLED: ${IPv6_ENABLED:-false} + +x-pysnmp_debug: &pysnmp_debug + PYSNMP_DEBUG: ${PYSNMP_DEBUG} + +x-dns_and_networks: &dns_and_networks + networks: + - sc4snmp_network + dns: + - ${COREDNS_ADDRESS} + +x-dependency_and_restart_policy: &dependency_and_restart_policy + depends_on: + - redis + - mongo + - coredns + restart: on-failure + +x-dependend_on_core_dns: &dependend_on_core_dns + depends_on: + - coredns + +secrets: {} + +networks: + sc4snmp_network: + name: sc4snmp_network + enable_ipv6: ${IPv6_ENABLED:-false} + ipam: + config: + - subnet: 172.28.0.0/16 + gateway: 172.28.0.1 + - subnet: fd02::/64 + gateway: fd02::1 + +services: + coredns: + image: ${COREDNS_IMAGE}:${COREDNS_TAG:-latest} + command: [-conf, /Corefile] + container_name: coredns + restart: on-failure + expose: + - '53' + - 53/udp + volumes: + - ${COREFILE_ABS_PATH}:/Corefile + networks: + sc4snmp_network: + ipv4_address: ${COREDNS_ADDRESS} + snmp-mibserver: + <<: [*dns_and_networks, *dependend_on_core_dns] + image: ${MIBSERVER_IMAGE}:${MIBSERVER_TAG:-latest} + container_name: snmp-mibserver + environment: + NGINX_ENTRYPOINT_QUIET_LOGS: ${NGINX_ENTRYPOINT_QUIET_LOGS:-1} + volumes: + - snmp-mibserver-tmp:/tmp/ + redis: + <<: [*dns_and_networks, *dependend_on_core_dns] + image: ${REDIS_IMAGE}:${REDIS_TAG:-latest} + container_name: redis + environment: + ALLOW_EMPTY_PASSWORD: yes + mongo: + <<: [*dns_and_networks, *dependend_on_core_dns] + image: ${MONGO_IMAGE}:${MONGO_TAG:-latest} + container_name: mongo + inventory: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-inventory + command: [inventory] + environment: + <<: *general_sc4snmp_data + LOG_LEVEL: ${INVENTORY_LOG_LEVEL:-INFO} + CHAIN_OF_TASKS_EXPIRY_TIME: ${CHAIN_OF_TASKS_EXPIRY_TIME:-500} + CONFIG_FROM_MONGO: ${CONFIG_FROM_MONGO:-false} + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - ${INVENTORY_FILE_ABSOLUTE_PATH}:/app/inventory/inventory.csv:ro + - inventory-pysnmp-cache-volume:/.pysnmp/:rw + - inventory-tmp:/tmp/:rw + scheduler: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + container_name: sc4snmp-scheduler + command: [celery, beat] + environment: + <<: *general_sc4snmp_data + LOG_LEVEL: ${SCHEDULER_LOG_LEVEL:-INFO} + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - scheduler-pysnmp-cache-volume:/.pysnmp/:rw + - scheduler-tmp:/tmp/:rw + traps: + <<: [*dns_and_networks, *dependency_and_restart_policy] + command: + - trap + container_name: sc4snmp-traps + environment: + <<: [*general_sc4snmp_data, *splunk_general_setup, *pysnmp_debug, *ipv6] + LOG_LEVEL: ${TRAP_LOG_LEVEL:-INFO} + SNMP_V3_SECURITY_ENGINE_ID: ${SNMP_V3_SECURITY_ENGINE_ID:-80003a8c04} + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + ports: + - mode: host + protocol: udp + published: ${TRAPS_PORT} + target: 2162 + - mode: host + protocol: udp + published: ${IPv6_TRAPS_PORT} + target: 2163 + volumes: + - ${TRAPS_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - traps-pysnmp-cache-volume:/.pysnmp/:rw + - traps-tmp:/tmp/:rw + worker-poller: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: + - celery + - worker-poller + deploy: + mode: replicated + replicas: ${WORKER_POLLER_REPLICAS:-2} + resources: + limits: + cpus: ${WORKER_POLLER_CPU_LIMIT:-0.50} + memory: ${WORKER_POLLER_MEMORY_LIMIT:-500M} + reservations: + cpus: ${WORKER_POLLER_CPU_RESERVATIONS:-0.25} + memory: ${WORKER_POLLER_MEMORY_RESERVATIONS:-250M} + environment: + <<: [*general_sc4snmp_data, *splunk_general_setup, *splunk_extended_setup, *workers_general_setup, + *pysnmp_debug, *ipv6] + WORKER_CONCURRENCY: ${WORKER_POLLER_CONCURRENCY:-2} + PREFETCH_COUNT: ${PREFETCH_POLLER_COUNT:-1} + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-poller-pysnmp-cache-volume:/.pysnmp/:rw + - worker-poller-tmp:/tmp/:rw + worker-sender: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, worker-sender] + environment: + <<: [*general_sc4snmp_data, *splunk_general_setup, *splunk_extended_setup, *workers_general_setup, + *pysnmp_debug] + # OTEL_METRICS_URL: #If sim enabled + # Workers configuration + WORKER_CONCURRENCY: ${WORKER_SENDER_CONCURRENCY:-2} + PREFETCH_COUNT: ${PREFETCH_SENDER_COUNT:-1} + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-sender-pysnmp-cache-volume:/.pysnmp/:rw + - worker-sender-tmp:/tmp/:rw + deploy: + mode: replicated + replicas: ${WORKER_SENDER_REPLICAS:-1} + resources: + limits: + cpus: ${WORKER_SENDER_CPU_LIMIT:-0.50} + memory: ${WORKER_SENDER_MEMORY_LIMIT:-500M} + reservations: + cpus: ${WORKER_SENDER_CPU_RESERVATIONS:-0.25} + memory: ${WORKER_SENDER_MEMORY_RESERVATIONS:-250M} + worker-trap: + <<: [*dns_and_networks, *dependency_and_restart_policy] + image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest} + command: [celery, worker-trap] + environment: + <<: [*general_sc4snmp_data, *splunk_general_setup, *splunk_extended_setup, *workers_general_setup, + *pysnmp_debug, *ipv6] + # OTEL_METRICS_URL: #If sim enabled + # Workers configuration + WORKER_CONCURRENCY: ${WORKER_TRAP_CONCURRENCY:-2} + PREFETCH_COUNT: ${PREFETCH_TRAP_COUNT:-1} + RESOLVE_TRAP_ADDRESS: ${RESOLVE_TRAP_ADDRESS:-false} + MAX_DNS_CACHE_SIZE_TRAPS: ${MAX_DNS_CACHE_SIZE_TRAPS:-500} + TTL_DNS_CACHE_TRAPS: ${TTL_DNS_CACHE_TRAPS:-1800} + volumes: + - ${SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH}:/app/config/config.yaml:ro + - worker-trap-pysnmp-cache-volume:/.pysnmp/:rw + - worker-trap-tmp:/tmp/:rw + deploy: + mode: replicated + replicas: ${WORKER_TRAP_REPLICAS:-2} + resources: + limits: + cpus: ${WORKER_TRAP_CPU_LIMIT:-0.50} + memory: ${WORKER_TRAP_MEMORY_LIMIT:-500M} + reservations: + cpus: ${WORKER_TRAP_CPU_RESERVATIONS:-0.25} + memory: ${WORKER_TRAP_MEMORY_RESERVATIONS:-250M} +volumes: + snmp-mibserver-tmp: + inventory-tmp: + inventory-pysnmp-cache-volume: + scheduler-tmp: + scheduler-pysnmp-cache-volume: + traps-pysnmp-cache-volume: + traps-tmp: + worker-poller-pysnmp-cache-volume: + worker-poller-tmp: + worker-sender-tmp: + worker-sender-pysnmp-cache-volume: + worker-trap-tmp: + worker-trap-pysnmp-cache-volume: diff --git a/docker_compose/manage_logs.py b/docker_compose/manage_logs.py index a7c28f675..7fee46eb1 100644 --- a/docker_compose/manage_logs.py +++ b/docker_compose/manage_logs.py @@ -1,12 +1,10 @@ import argparse import os -import re from typing import Union import ruamel.yaml -DEPENDENCIES = ["snmp-mibserver", "redis", "mongo"] -DOCKER_COMPOSE_DEPENDENCIES = "docker-compose-dependencies.yaml" +DOCKER_COMPOSE = "docker-compose.yaml" def human_bool(flag: Union[str, bool], default: bool = False) -> bool: @@ -74,90 +72,35 @@ def load_template(environment: dict, service_name: str) -> dict: def create_logs(environment, path_to_compose_files): - files_list = os.listdir(path_to_compose_files) - compose_files = [ - f - for f in files_list - if re.match(r"docker-compose-(?!dependencies|network|secrets).*.yaml", f) - ] - - for filename in compose_files: - service_name = filename.removeprefix("docker-compose-").removesuffix(".yaml") - template_yaml = load_template(environment, service_name) - try: - yaml = ruamel.yaml.YAML() - with open(os.path.join(path_to_compose_files, filename)) as file: - yaml_file = yaml.load(file) - yaml_file["services"][service_name].update(template_yaml) - - with open(os.path.join(path_to_compose_files, filename), "w") as file: - yaml.dump(yaml_file, file) - except Exception as e: - print( - f"Problem with editing docker-compose-{service_name}.yaml. Error: {e}" - ) - try: - yaml2 = ruamel.yaml.YAML() - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_DEPENDENCIES) - ) as file: - yaml_file = yaml2.load(file) + yaml = ruamel.yaml.YAML() + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE)) as file: + yaml_file = yaml.load(file) - for service_name in DEPENDENCIES: + for service_name in yaml_file["services"].keys(): template_yaml = load_template(environment, service_name) yaml_file["services"][service_name].update(template_yaml) - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_DEPENDENCIES), "w" - ) as file: - yaml2.dump(yaml_file, file) + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE), "w") as file: + yaml.dump(yaml_file, file) except Exception as e: - print(f"Problem with editing docker-compose-dependencies.yaml. Error: {e}") + print(f"Problem with editing docker-compose.yaml. Error: {e}") def delete_logs(path_to_compose_files): - files_list = os.listdir(path_to_compose_files) - compose_files = [ - f - for f in files_list - if re.match(r"docker-compose-(?!dependencies|network|secrets).*.yaml", f) - ] - - for filename in compose_files: - service_name = filename.removeprefix("docker-compose-").removesuffix(".yaml") - try: - with open(os.path.join(path_to_compose_files, filename)) as file: - yaml = ruamel.yaml.YAML() - yaml_file = yaml.load(file) - - yaml_file["services"][service_name]["logging"]["driver"] = "json-file" - yaml_file["services"][service_name]["logging"].pop("options") - - with open(os.path.join(path_to_compose_files, filename), "w") as file: - yaml.dump(yaml_file, file) - except Exception as e: - print( - f"Problem with editing docker-compose-{service_name}.yaml. Error: {e}" - ) - try: - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_DEPENDENCIES) - ) as file: - yaml2 = ruamel.yaml.YAML() - yaml_file = yaml2.load(file) + yaml = ruamel.yaml.YAML() + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE)) as file: + yaml_file = yaml.load(file) - for service_name in DEPENDENCIES: + for service_name in yaml_file["services"].keys(): yaml_file["services"][service_name]["logging"]["driver"] = "json-file" yaml_file["services"][service_name]["logging"].pop("options") - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_DEPENDENCIES), "w" - ) as file: - yaml2.dump(yaml_file, file) + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE), "w") as file: + yaml.dump(yaml_file, file) except Exception as e: - print(f"Problem with editing docker-compose-dependencies.yaml. Error: {e}") + print(f"Problem with editing docker-compose.yaml. Error: {e}") def main(): diff --git a/docker_compose/manage_secrets.py b/docker_compose/manage_secrets.py index 89d1e274a..079d8890f 100644 --- a/docker_compose/manage_secrets.py +++ b/docker_compose/manage_secrets.py @@ -2,11 +2,10 @@ import os from typing import Union -import yaml +import ruamel.yaml -DOCKER_COMPOSE_SECRETS = "docker-compose-secrets.yaml" -DOCKER_COMPOSE_WORKER_POLLER = "docker-compose-worker-poller.yaml" -DOCKER_COMPOSE_TRAPS = "docker-compose-traps.yaml" +SERVICE_SECRETS = ["worker-poller", "traps"] +DOCKER_COMPOSE = "docker-compose.yaml" def human_bool(flag: Union[str, bool], default: bool = False) -> bool: @@ -80,125 +79,97 @@ def create_secrets( try: # Load docker-compose-secrets.yaml to a dictionary and update "secrets" section. If the same secret # has been already configured, stop processing further. - with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE_SECRETS)) as file: - secrets_file = yaml.load(file, Loader=yaml.FullLoader) - if secrets_file["secrets"] is None or "secrets" not in secrets_file: - secrets_file["secrets"] = {} + yaml = ruamel.yaml.YAML() + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE)) as file: + yaml_file = yaml.load(file) + if yaml_file["secrets"] is None or "secrets" not in yaml_file: + yaml_file["secrets"] = {} for new_secret in new_secrets: - if new_secret["secret_name"] in secrets_file["secrets"]: + if new_secret["secret_name"] in yaml_file["secrets"]: print(f"Secret {secret_name} already configured. New secret not added.") return - secrets_file["secrets"][new_secret["secret_name"]] = new_secret[ + yaml_file["secrets"][new_secret["secret_name"]] = new_secret[ "secret_config" ] - secrets_file_ready = True - except Exception: - print("Problem with editing docker-compose-secrets.yaml. Secret not added.") - secrets_file_ready = False + secrets_ready = True - if make_change_in_worker_poller: - worker_poller_file, worker_poller_file_ready = load_compose_worker_poller( - new_secrets_in_workers, path_to_compose_files - ) - else: - worker_poller_file = {} - worker_poller_file_ready = True + if make_change_in_worker_poller: + yaml_file, worker_poller_ready = load_compose_worker_poller( + new_secrets_in_workers, yaml_file + ) + else: + worker_poller_ready = True - if make_change_in_traps: - traps_file, traps_file_ready = load_compose_traps( - new_secrets_in_workers, path_to_compose_files + if make_change_in_traps: + yaml_file, traps_ready = load_compose_traps( + new_secrets_in_workers, yaml_file + ) + else: + traps_ready = True + + save_to_compose_files( + path_to_compose_files, + secret_name, + yaml_file, + secrets_ready, + traps_ready, + variables, + worker_poller_ready, ) - else: - traps_file = {} - traps_file_ready = True - - save_to_compose_files( - make_change_in_traps, - make_change_in_worker_poller, - path_to_compose_files, - secret_name, - secrets_file, - secrets_file_ready, - traps_file, - traps_file_ready, - variables, - worker_poller_file, - worker_poller_file_ready, - ) + except Exception as e: + print(f"Problem with adding secrets. Error: {e}") def save_to_compose_files( - make_change_in_traps, - make_change_in_worker_poller, path_to_compose_files, secret_name, - secrets_file, - secrets_file_ready, - traps_file, - traps_file_ready, + yaml_file, + secrets_ready, + traps_ready, variables, - worker_poller_file, - worker_poller_file_ready, + worker_poller_ready, ): - if secrets_file_ready and worker_poller_file_ready and traps_file_ready: + if secrets_ready and worker_poller_ready and traps_ready: # If all three files were loaded into dictionary and updated successfully, # save the latest configuration to files. - save_to_yaml_file(path_to_compose_files, DOCKER_COMPOSE_SECRETS, secrets_file) with open(os.path.join(path_to_compose_files, ".env"), "a") as file: for k, v in variables.items(): if v: file.write(f"\n{secret_name}_{k}={v}") - if make_change_in_worker_poller: - save_to_yaml_file( - path_to_compose_files, DOCKER_COMPOSE_WORKER_POLLER, worker_poller_file - ) - if make_change_in_traps: - save_to_yaml_file(path_to_compose_files, DOCKER_COMPOSE_TRAPS, traps_file) - -def save_to_yaml_file(file_path, file_name, file_content): - with open(os.path.join(file_path, file_name), "w") as file: - yaml.dump(file_content, file, default_flow_style=False) + yaml = ruamel.yaml.YAML() + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE), "w") as file: + yaml.dump(yaml_file, file) -def load_compose_traps(new_secrets_in_workers, path_to_compose_files): +def load_compose_traps(new_secrets_in_workers, yaml_file): # If the secret should be added to traps, load docker-compose-traps.yaml to a dictionary and # update "secrets" section. try: - with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE_TRAPS)) as file: - traps_file = yaml.load(file, Loader=yaml.FullLoader) - if "secrets" not in traps_file["services"]["traps"]: - traps_file["services"]["traps"]["secrets"] = [] - traps_file["services"]["traps"]["secrets"].extend(new_secrets_in_workers) - traps_file_ready = True - except Exception: - print("Problem with editing docker-compose-traps.yaml. Secret not added.") - traps_file = {} - traps_file_ready = False - return traps_file, traps_file_ready + if "secrets" not in yaml_file["services"]["traps"]: + yaml_file["services"]["traps"]["secrets"] = [] + yaml_file["services"]["traps"]["secrets"].extend(new_secrets_in_workers) + traps_ready = True + except Exception as e: + print(f"Problem with editing traps. Secret not added. Error {e}") + yaml_file = {} + traps_ready = False + return yaml_file, traps_ready -def load_compose_worker_poller(new_secrets_in_workers, path_to_compose_files): +def load_compose_worker_poller(new_secrets_in_workers, yaml_file): # If the secret should be added to worker poller, load docker-compose-worker-poller.yaml to a dictionary and # update "secrets" section. try: - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_WORKER_POLLER) - ) as file: - worker_poller_file = yaml.load(file, Loader=yaml.FullLoader) - if "secrets" not in worker_poller_file["services"]["worker-poller"]: - worker_poller_file["services"]["worker-poller"]["secrets"] = [] - worker_poller_file["services"]["worker-poller"]["secrets"].extend( - new_secrets_in_workers - ) - worker_poller_file_ready = True + if "secrets" not in yaml_file["services"]["worker-poller"]: + yaml_file["services"]["worker-poller"]["secrets"] = [] + yaml_file["services"]["worker-poller"]["secrets"].extend(new_secrets_in_workers) + worker_poller_ready = True except Exception: - print( - "Problem with editing docker-compose-worker-poller.yaml. Secret not added." - ) - worker_poller_file = {} - worker_poller_file_ready = False - return worker_poller_file, worker_poller_file_ready + print("Problem with editing worker-poller. Secret not added.") + yaml_file = {} + worker_poller_ready = False + return yaml_file, worker_poller_ready def store_secrets(secret_name, variables): @@ -242,45 +213,41 @@ def delete_secrets( for key in variables.keys(): secrets.append(f"{secret_name}_{key}") - secrets_file = load_compose_secrets(path_to_compose_files, secrets) + yaml = ruamel.yaml.YAML() + try: + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE)) as file: + yaml_file = yaml.load(file) - # Save the updated docker-compose-secrets.yaml configuration - save_to_yaml_file(path_to_compose_files, DOCKER_COMPOSE_SECRETS, secrets_file) + yaml_file = load_compose_secrets(yaml_file, secrets) + # Save the updated docker-compose-secrets.yaml configuration - # Delete secrets from .env - delete_secrets_from_env(path_to_compose_files, secrets) + if make_change_in_worker_poller: + # filter out secrets destined for deletion - if make_change_in_worker_poller: - # Load docker-compose-worker-poller.yaml to dictionary and filter out secrets destined for deletion - with open( - os.path.join(path_to_compose_files, DOCKER_COMPOSE_WORKER_POLLER) - ) as file: - worker_poller_file = yaml.load(file, Loader=yaml.FullLoader) - worker_poller_file["services"]["worker-poller"]["secrets"] = list( - filter( - lambda el: el["source"] not in secrets, - worker_poller_file["services"]["worker-poller"]["secrets"], + yaml_file["services"]["worker-poller"]["secrets"] = list( + filter( + lambda el: el["source"] not in secrets, + yaml_file["services"]["worker-poller"]["secrets"], + ) ) - ) - # Save updated docker-compose-worker-poller.yaml configuration - save_to_yaml_file( - path_to_compose_files, DOCKER_COMPOSE_WORKER_POLLER, worker_poller_file - ) - - if make_change_in_traps: - # Load docker-compose-traps.yaml to dictionary and filter out secrets destined for deletion - with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE_TRAPS)) as file: - traps_file = yaml.load(file, Loader=yaml.FullLoader) - traps_file["services"]["traps"]["secrets"] = list( - filter( - lambda el: el["source"] not in secrets, - traps_file["services"]["traps"]["secrets"], + if make_change_in_traps: + # Load docker-compose-traps.yaml to dictionary and filter out secrets destined for deletion + yaml_file["services"]["traps"]["secrets"] = list( + filter( + lambda el: el["source"] not in secrets, + yaml_file["services"]["traps"]["secrets"], + ) ) - ) - # Save updated docker-compose-traps.yaml configuration - save_to_yaml_file(path_to_compose_files, DOCKER_COMPOSE_TRAPS, traps_file) + except Exception as e: + print(f"Problem with editing secrets section. Secret not added. Error: {e}") + + with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE), "w") as file: + yaml.dump(yaml_file, file) + + # Delete secrets from .env + delete_secrets_from_env(path_to_compose_files, secrets) def delete_secrets_from_env(path_to_compose_files, secrets): @@ -307,14 +274,12 @@ def delete_secrets_from_env(path_to_compose_files, secrets): print(f"Error: {e}") -def load_compose_secrets(path_to_compose_files, secrets): +def load_compose_secrets(yaml_file, secrets): # Load docker-compose-secrets.yaml file to a dictionary and delete desired secrets - with open(os.path.join(path_to_compose_files, DOCKER_COMPOSE_SECRETS)) as file: - secrets_file = yaml.load(file, Loader=yaml.FullLoader) for secret in secrets: - if secret in secrets_file["secrets"]: - del secrets_file["secrets"][secret] - return secrets_file + if secret in yaml_file["secrets"]: + del yaml_file["secrets"][secret] + return yaml_file def main(): diff --git a/docs/dockercompose/10-enable-ipv6.md b/docs/dockercompose/10-enable-ipv6.md index 93e9f305a..82ff73301 100644 --- a/docs/dockercompose/10-enable-ipv6.md +++ b/docs/dockercompose/10-enable-ipv6.md @@ -10,7 +10,7 @@ To avoid any problem with configuring the network, it is recommended to use the To enable IPv6 for SC4SNMP, set `IPv6_ENABLED` variable to `true` in `.env` file. The default subnet used for SC4SNMP network in docker is `fd02::/64`, this and other network configuration can be -changed in the `docker-compose-network.yml` file. +changed in the `docker-compose.yaml` file in `networks` section. Default trap port for notifications for IPv6 is `2163`. You can change it to any other port if needed with `IPv6_TRAPS_PORT` parameter in `.env` file. The IPv6 port and IPv4 port cannot be the same. diff --git a/docs/dockercompose/2-download-package.md b/docs/dockercompose/2-download-package.md index cd23b953a..6a00101aa 100644 --- a/docs/dockercompose/2-download-package.md +++ b/docs/dockercompose/2-download-package.md @@ -13,9 +13,13 @@ After configuration, application can be deployed by running the following command inside the `docker_compose` directory: ```shell -sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +sudo docker compose up -d ``` +!!! info + The installation process changed from version **1.12.1**. For lower version refer to the corresponding + documentation. + The same command can be run to apply any updated configuration changes. ## Uninstall the app @@ -23,5 +27,5 @@ The same command can be run to apply any updated configuration changes. To uninstall the app, run the following command inside the `docker_compose` directory: ```shell -sudo docker compose $(find docker* | sed -e 's/^/-f /') down +sudo docker compose down ``` \ No newline at end of file diff --git a/docs/dockercompose/5-traps-configuration.md b/docs/dockercompose/5-traps-configuration.md index 9ec5edf14..e601c44e0 100644 --- a/docs/dockercompose/5-traps-configuration.md +++ b/docs/dockercompose/5-traps-configuration.md @@ -1,4 +1,3 @@ -# Traps configuration Scheduler configuration is stored in the `traps-config.yaml` file. This file has the following sections: diff --git a/docs/dockercompose/7-snmpv3-secrets.md b/docs/dockercompose/7-snmpv3-secrets.md index 714f8ea39..e29e8ed65 100644 --- a/docs/dockercompose/7-snmpv3-secrets.md +++ b/docs/dockercompose/7-snmpv3-secrets.md @@ -3,6 +3,13 @@ Creating a secret requires updating configuration of several docker compose files. To simplify this process, inside the `docker_compose` package there is a `manage_secrets.py` file which will automatically manage secrets. +## Prerequisites + +Running script requires installation of `ruamel.yaml` package for python. It can be done with command: +``` +pip3 install ruamel.yaml +``` + ## Creating a new secret To create a new secret, `manage_secrets.py` must be run with the following flags: @@ -23,7 +30,7 @@ To create a new secret, `manage_secrets.py` must be run with the following flags This script, apart from updating configuration files, creates environmental variables with values of the secret at the end of the `.env` file in the `docker_compose` directory. To apply those secrets run the -`sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d` command inside the `docker_compose` directory. After execution of the command, plain text secrets +`sudo docker compose up -d` command inside the `docker_compose` directory. After execution of the command, plain text secrets from the `.env` file can be deleted. > **_NOTE:_** In case of any changes in `.env`, the secrets must be recreated by [deleting](#deleting-a-secret) any > previously existing secrets and creating them once again. Changes in `.env` include creating new secrets. @@ -43,7 +50,7 @@ python3 --path_to_compose \ Inside `docker_compose` directory run: ```shell -sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +sudo docker compose up -d ``` Now, the following lines from the `.env` can be deleted: diff --git a/docs/dockercompose/9-splunk-logging.md b/docs/dockercompose/9-splunk-logging.md index 7de23c2e3..05fc2e461 100644 --- a/docs/dockercompose/9-splunk-logging.md +++ b/docs/dockercompose/9-splunk-logging.md @@ -37,7 +37,7 @@ python3 manage_logs.py --path_to_compose /home/ubuntu/docker_compose --enable_lo The script will add required configuration for logging under services in docker compose files. To apply the changes run the: ``` -sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +sudo docker compose up -d ``` command inside the `docker_compose` directory. @@ -59,7 +59,7 @@ python3 manage_logs.py --path_to_compose /home/ubuntu/docker_compose --disable_l To apply the changes run the: ``` -sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +sudo docker compose up -d ``` command inside the `docker_compose` directory. diff --git a/docs/microk8s/configuration/trap-configuration.md b/docs/microk8s/configuration/trap-configuration.md index e12a20879..cc36c28e6 100644 --- a/docs/microk8s/configuration/trap-configuration.md +++ b/docs/microk8s/configuration/trap-configuration.md @@ -85,6 +85,19 @@ The following is an example of an SNMPv3 trap: snmptrap -v3 -e 80003a8c04 -l authPriv -u snmp-poller -a SHA -A PASSWORD1 -x AES -X PASSWORD1 10.202.13.233 '' 1.3.6.1.2.1.2.2.1.1.1 ``` +### Updating trap configuration +If you need to update part of the traps configuration, you can do it by editing the `values.yaml` and then running the following command to restart the pod deployment: +``` +microk8s kubectl rollout restart deployment snmp-splunk-connect-for-snmp-trap -n sc4snmp +``` + +!!! info + The name of the deployment can differ based on the helm installation name. + This can be checked with the following command: + ``` + microk8s kubectl get deployments -n sc4snmp + ``` + ### Define external gateway for traps If you use SC4SNMP on a single machine, configure `loadBalancerIP`. @@ -150,17 +163,4 @@ In case you want to see traps events collected as one event inside Splunk, you c traps: aggregateTrapsEvents: "true" ``` -After that run the upgrade command. - -### Updating trap configuration -If you need to update part of the traps configuration that changes the configmap, you can do it by editing the `values. -yaml` and then running the following command to restart the pod deployment: -``` -microk8s kubectl rollout restart deployment snmp-splunk-connect-for-snmp-trap -n sc4snmp -``` - -!!! info - The name of the deployment can differ based on the helm installation name. This can be checked with the following command: - ``` - microk8s kubectl get deployments -n sc4snmp - ``` +Then the upgrade command can be executed. \ No newline at end of file diff --git a/integration_tests/automatic_setup_compose.sh b/integration_tests/automatic_setup_compose.sh index 0bc47fcfc..54463c5fe 100755 --- a/integration_tests/automatic_setup_compose.sh +++ b/integration_tests/automatic_setup_compose.sh @@ -182,6 +182,7 @@ awk -v scheduler_path="$SCHEDULER_CONFIG_FILE_ABSOLUTE_PATH" \ mv "$TEMP_ENV_FILE" .env # Create snmpv3 secret +python3 -m pip install ruamel.yaml python3 $(realpath "manage_secrets.py") --path_to_compose $(pwd) \ --secret_name sv3poller \ --userName r-wuser \ @@ -201,9 +202,8 @@ sudo docker run -d -p 1164:161/udp tandrup/snmpsim sudo docker run -d -p 1165:161/udp tandrup/snmpsim sudo docker run -d -p 1166:161/udp -v $(pwd)/snmpsim/data:/usr/local/snmpsim/data -e EXTRA_FLAGS="--variation-modules-dir=/usr/local/snmpsim/variation --data-dir=/usr/local/snmpsim/data" tandrup/snmpsim - echo $(green "Running up Docker Compose environment") -sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d +sudo docker compose up -d wait_for_containers_to_be_up sudo docker ps diff --git a/integration_tests/splunk_test_utils.py b/integration_tests/splunk_test_utils.py index f48f1efac..4705538cc 100644 --- a/integration_tests/splunk_test_utils.py +++ b/integration_tests/splunk_test_utils.py @@ -137,7 +137,7 @@ def update_traps_secrets_compose(secrets): def upgrade_docker_compose(): - os.system("sudo docker compose $(find docker* | sed -e 's/^/-f /') up -d") + os.system("sudo docker compose up -d") def create_v3_secrets_compose(