From dab1777178dde27145cd2a28d5b576840ed482eb Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Sun, 30 Jun 2024 01:00:46 +0200 Subject: [PATCH 1/9] * Introduced a new attribute, webhook_override, to the MsSink class * Created MsTeamsWebhookUrlTransformer to enable overriding the webhook_url using annotations from yaml files * Enhanced MsTeamsSender.send_finding_to_ms_teams to support overriding the webhook_url * Moved shared methods and logic from ChannelTransformer to BaseChannelTransformer --- .../core/sinks/common/channel_transformer.py | 62 +++++++++---------- .../core/sinks/msteams/msteams_sink.py | 3 +- .../core/sinks/msteams/msteams_sink_params.py | 10 +++ .../msteams/msteams_webhook_tranformer.py | 41 ++++++++++++ src/robusta/integrations/msteams/sender.py | 12 +++- 5 files changed, 93 insertions(+), 35 deletions(-) create mode 100644 src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py diff --git a/src/robusta/core/sinks/common/channel_transformer.py b/src/robusta/core/sinks/common/channel_transformer.py index a6300fb45..bec5c3ce9 100644 --- a/src/robusta/core/sinks/common/channel_transformer.py +++ b/src/robusta/core/sinks/common/channel_transformer.py @@ -1,6 +1,6 @@ from collections import defaultdict from string import Template -from typing import Dict, Optional, Union +from typing import Dict, Optional import regex @@ -18,21 +18,7 @@ MISSING = "" -class ChannelTransformer: - @classmethod - def validate_channel_override(cls, v: Union[str, None]): - if v: - if regex.match(ONLY_VALUE_PATTERN, v): - return "$" + v - if not regex.match(COMPOSITE_PATTERN, v): - err_msg = ( - f"channel_override must be '{CLUSTER_PREF}' or '{LABELS_PREF}foo' or '{ANNOTATIONS_PREF}foo' " - f"or contain patters like: '${CLUSTER_PREF}'/'${LABELS_PREF}foo'/" - f"'${ANNOTATIONS_PREF}foo'" - ) - raise ValueError(err_msg) - return v - +class BaseChannelTransformer: @classmethod def normalize_key_string(cls, s: str) -> str: return s.replace("/", "_").replace(".", "_").replace("-", "_") @@ -47,7 +33,7 @@ def normalize_dict_keys(cls, metadata: Dict) -> Dict: # else, if found, return replacement else return MISSING @classmethod def get_replacement(cls, prefix: str, value: str, normalized_replacements: Dict) -> str: - if prefix in value: # value is in the format of "$prefix" or "prefix" + if prefix in value: value = cls.normalize_key_string(value.replace(prefix, "")) if "$" in value: return Template(value).safe_substitute(normalized_replacements) @@ -56,13 +42,7 @@ def get_replacement(cls, prefix: str, value: str, normalized_replacements: Dict) return "" @classmethod - def replace_token( - cls, - pattern: regex.Pattern, - prefix: str, - channel: str, - replacements: Dict[str, str], - ) -> str: + def replace_token(cls, pattern: regex.Pattern, prefix: str, channel: str, replacements: Dict[str, str]) -> str: tokens = pattern.findall(channel) for token in tokens: clean_token = token.replace("{", "").replace("}", "") @@ -71,6 +51,30 @@ def replace_token( channel = channel.replace(token, replacement) return channel + @classmethod + def process_template_annotations(cls, channel: str, annotations: Dict[str, str]) -> str: + if ANNOTATIONS_PREF in channel: + normalized_annotations = cls.normalize_dict_keys(annotations) + channel = cls.replace_token(BRACKETS_PATTERN, ANNOTATIONS_PREF, channel, normalized_annotations) + channel = cls.replace_token(ANNOTATIONS_PREF_PATTERN, ANNOTATIONS_PREF, channel, normalized_annotations) + return channel + + +class ChannelTransformer(BaseChannelTransformer): + @classmethod + def validate_channel_override(cls, v: Optional[str]) -> str: + if v: + if regex.match(ONLY_VALUE_PATTERN, v): + return "$" + v + if not regex.match(COMPOSITE_PATTERN, v): + err_msg = ( + f"channel_override must be '{CLUSTER_PREF}' or '{LABELS_PREF}foo' or '{ANNOTATIONS_PREF}foo' " + f"or contain patters like: '${CLUSTER_PREF}'/'${LABELS_PREF}foo'/" + f"'${ANNOTATIONS_PREF}foo'" + ) + raise ValueError(err_msg) + return v + @classmethod def template( cls, @@ -93,14 +97,6 @@ def template( channel = cls.replace_token(BRACKETS_PATTERN, LABELS_PREF, channel, normalized_labels) channel = cls.replace_token(LABEL_PREF_PATTERN, LABELS_PREF, channel, normalized_labels) - if ANNOTATIONS_PREF in channel: - normalized_annotations = cls.normalize_dict_keys(annotations) - channel = cls.replace_token(BRACKETS_PATTERN, ANNOTATIONS_PREF, channel, normalized_annotations) - channel = cls.replace_token( - ANNOTATIONS_PREF_PATTERN, - ANNOTATIONS_PREF, - channel, - normalized_annotations, - ) + channel = cls.process_template_annotations(channel, annotations) return channel if MISSING not in channel else default_channel diff --git a/src/robusta/core/sinks/msteams/msteams_sink.py b/src/robusta/core/sinks/msteams/msteams_sink.py index c1783260c..423691ad4 100644 --- a/src/robusta/core/sinks/msteams/msteams_sink.py +++ b/src/robusta/core/sinks/msteams/msteams_sink.py @@ -8,8 +8,9 @@ class MsTeamsSink(SinkBase): def __init__(self, sink_config: MsTeamsSinkConfigWrapper, registry): super().__init__(sink_config.ms_teams_sink, registry) self.webhook_url = sink_config.ms_teams_sink.webhook_url + self.webhook_override = sink_config.ms_teams_sink.webhook_override def write_finding(self, finding: Finding, platform_enabled: bool): MsTeamsSender.send_finding_to_ms_teams( - self.webhook_url, finding, platform_enabled, self.cluster_name, self.account_id + self.webhook_url, finding, platform_enabled, self.cluster_name, self.account_id, self.webhook_override ) diff --git a/src/robusta/core/sinks/msteams/msteams_sink_params.py b/src/robusta/core/sinks/msteams/msteams_sink_params.py index 826c0ce27..85f016319 100644 --- a/src/robusta/core/sinks/msteams/msteams_sink_params.py +++ b/src/robusta/core/sinks/msteams/msteams_sink_params.py @@ -1,14 +1,24 @@ +from typing import Optional + +from pydantic import validator + +from robusta.core.sinks.msteams.msteams_webhook_tranformer import MsTeamsWebhookUrlTransformer from robusta.core.sinks.sink_base_params import SinkBaseParams from robusta.core.sinks.sink_config import SinkConfigBase class MsTeamsSinkParams(SinkBaseParams): webhook_url: str + webhook_override: Optional[str] = None @classmethod def _get_sink_type(cls): return "msteams" + @validator("webhook_override") + def validate_webhook_override(cls, v: str): + return MsTeamsWebhookUrlTransformer.validate_channel_override(v) + class MsTeamsSinkConfigWrapper(SinkConfigBase): ms_teams_sink: MsTeamsSinkParams diff --git a/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py new file mode 100644 index 000000000..4e4dd6bf8 --- /dev/null +++ b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py @@ -0,0 +1,41 @@ +from typing import Dict, Optional + +import regex + +from robusta.core.sinks.common.channel_transformer import ANNOTATIONS_PREF, MISSING, BaseChannelTransformer + +ANNOTATIONS_COMPOSITE_PATTERN = r".*\$({?annotations.[^$]+).*" +ANNOTATIONS_ONLY_VALUE_PATTERN = r"^(annotations.[^$]+)$" + + +# This class supports overriding the webhook_url only using annotations from yaml files. +# Annotations are used instead of labels because urls can be passed to annotations contrary to labels. +# Labels must be an empty string or consist of alphanumeric characters, '-', '_', or '.', +# and must start and end with an alphanumeric character (e.g., 'MyValue', 'my_value', or '12345'). +# The regex used for label validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?'. +class MsTeamsWebhookUrlTransformer(BaseChannelTransformer): + @classmethod + def validate_webhook_override(cls, v: Optional[str]): + if v: + if regex.match(ANNOTATIONS_ONLY_VALUE_PATTERN, v): + return "$" + v + if not regex.match(ANNOTATIONS_COMPOSITE_PATTERN, v): + err_msg = f"webhook_override must be '{ANNOTATIONS_PREF}foo' or contain patterns like: '${ANNOTATIONS_PREF}foo'" + raise ValueError(err_msg) + return v + + @classmethod + def template( + cls, + webhook_override: Optional[str], + default_webhook_url: str, + annotations: Dict[str, str], + ) -> str: + if not webhook_override: + return default_webhook_url + + webhook_url = webhook_override + + webhook_url = cls.process_template_annotations(webhook_url, annotations) + + return webhook_url if MISSING not in webhook_url else default_webhook_url diff --git a/src/robusta/integrations/msteams/sender.py b/src/robusta/integrations/msteams/sender.py index 96c74f557..1e2db511c 100644 --- a/src/robusta/integrations/msteams/sender.py +++ b/src/robusta/integrations/msteams/sender.py @@ -13,6 +13,7 @@ MarkdownBlock, TableBlock, ) +from robusta.core.sinks.msteams.msteams_webhook_tranformer import MsTeamsWebhookUrlTransformer from robusta.integrations.msteams.msteams_msg import MsTeamsMsg @@ -50,8 +51,17 @@ def __split_block_to_files_and_all_the_rest(cls, enrichment: Enrichment): @classmethod def send_finding_to_ms_teams( - cls, webhook_url: str, finding: Finding, platform_enabled: bool, cluster_name: str, account_id: str + cls, + webhook_url: str, + finding: Finding, + platform_enabled: bool, + cluster_name: str, + account_id: str, + webhook_override: str, ): + webhook_url = MsTeamsWebhookUrlTransformer.template( + webhook_override=webhook_override, default_webhook_url=webhook_url, annotations=finding.subject.annotations + ) msg = MsTeamsMsg(webhook_url) msg.write_title_and_desc(platform_enabled, finding, cluster_name, account_id) From d61e2d4d70d12a8e349104e85973d4fbf5f0a61f Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Sun, 30 Jun 2024 01:08:57 +0200 Subject: [PATCH 2/9] * Added tests for MsTeamsWebhookUrlTransformer class based on tests from test_channel_transformer.py --- tests/test_ms_teams_transformer.py | 110 +++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/test_ms_teams_transformer.py diff --git a/tests/test_ms_teams_transformer.py b/tests/test_ms_teams_transformer.py new file mode 100644 index 000000000..0376b0600 --- /dev/null +++ b/tests/test_ms_teams_transformer.py @@ -0,0 +1,110 @@ +import logging + +import pytest + +from robusta.core.sinks.msteams.msteams_webhook_tranformer import MsTeamsWebhookUrlTransformer + +OVERRIDE_WEBHOOK = "override-webhook-url" +DEFAULT_WEBHOOK = "default-webhook-url" + + +testdata_template = [ + ( + "annotations.msteams", + {"msteams": OVERRIDE_WEBHOOK}, + OVERRIDE_WEBHOOK, + "override channel not found", + ), + ( + "annotations.msteams", + {"msteam": OVERRIDE_WEBHOOK}, + DEFAULT_WEBHOOK, + "override - default channel not found", + ), + ( + "$annotations.msteams", + {"msteams": OVERRIDE_WEBHOOK}, + OVERRIDE_WEBHOOK, + "override - default channel not chosen", + ), + ( + "$annotations.msteams", + {"variable": OVERRIDE_WEBHOOK}, + DEFAULT_WEBHOOK, + "override - default channel not chosen", + ), + ( + "${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": OVERRIDE_WEBHOOK}, + OVERRIDE_WEBHOOK, + "override channel not found", + ), + ( + "${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service": OVERRIDE_WEBHOOK}, + DEFAULT_WEBHOOK, + "override - default channel not chosen", + ), + ( + "${annotations.kubernetes.io/service-name}", + {}, + DEFAULT_WEBHOOK, + "override - default channel not chosen", + ), + # webhook_override: "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}" + ( + "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": "yyy"}, + DEFAULT_WEBHOOK, + "override channel not found", + ), + ( + "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": "yyy"}, + DEFAULT_WEBHOOK, + "override - default channel not chosen", + ), +] + + +@pytest.mark.parametrize("webhook_override, annotations, expected, error", testdata_template) +def test_ms_teams_template(webhook_override, annotations, expected, error): + logging.info(f"testing {webhook_override}") + webhook_url = MsTeamsWebhookUrlTransformer.template( + webhook_override=webhook_override, + default_webhook_url=DEFAULT_WEBHOOK, + annotations=annotations, + ) + assert webhook_url == expected, f"{webhook_override} {error}" + + +testdata_validate = [ + ("annotations.team", "$annotations.team", "missing '$' prefix"), + ("$annotations.team", "$annotations.team", "override should be left unchanged"), + ("${annotations.team}", "${annotations.team}", "override should be left unchanged"), +] + + +@pytest.mark.parametrize("webhook_override, expected, error", testdata_validate) +def test_ms_teams_validate_webhook_override(webhook_override, expected, error): + logging.info(f"testing {webhook_override}") + webhook_url = MsTeamsWebhookUrlTransformer.validate_webhook_override(webhook_override) + assert webhook_url == expected, f"{webhook_override} {error}" + + +testdata_should_throw = [ + "cluste_name", + "annotations.", + "$annotations.", + "invalid.something", + "labels.", + "$labels.", + "test", +] + + +@pytest.mark.parametrize("webhook_override", testdata_should_throw) +def test_ms_teams_validate_webhook_override_should_throw(webhook_override): + logging.info(f"testing {webhook_override}") + with pytest.raises(ValueError): + MsTeamsWebhookUrlTransformer.validate_webhook_override(webhook_override) From 93862442e8566361bdf8bda47d47389c4a5163e2 Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Sun, 30 Jun 2024 12:16:03 +0200 Subject: [PATCH 3/9] * Fixed MsTeamsSinkParams's validate_webhook_override method --- src/robusta/core/sinks/msteams/msteams_sink_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robusta/core/sinks/msteams/msteams_sink_params.py b/src/robusta/core/sinks/msteams/msteams_sink_params.py index 85f016319..b10ba7ea0 100644 --- a/src/robusta/core/sinks/msteams/msteams_sink_params.py +++ b/src/robusta/core/sinks/msteams/msteams_sink_params.py @@ -17,7 +17,7 @@ def _get_sink_type(cls): @validator("webhook_override") def validate_webhook_override(cls, v: str): - return MsTeamsWebhookUrlTransformer.validate_channel_override(v) + return MsTeamsWebhookUrlTransformer.validate_webhook_override(v) class MsTeamsSinkConfigWrapper(SinkConfigBase): From d561722bb9074c57eadda8c7138e3f4c5c18df2f Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Tue, 2 Jul 2024 14:52:47 +0200 Subject: [PATCH 4/9] * Added possibility to override webhook_url value for ms_teams_sink from additional_env_vars * Added additional pytests for MsTeamsWebhookUrlTransformer to test_ms_teams_transformer.py file --- .../msteams/msteams_webhook_tranformer.py | 28 +- tests/test_ms_teams_transformer.py | 239 +++++++++++------- 2 files changed, 173 insertions(+), 94 deletions(-) diff --git a/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py index 4e4dd6bf8..851756118 100644 --- a/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py +++ b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py @@ -1,3 +1,4 @@ +import os from typing import Dict, Optional import regex @@ -6,6 +7,16 @@ ANNOTATIONS_COMPOSITE_PATTERN = r".*\$({?annotations.[^$]+).*" ANNOTATIONS_ONLY_VALUE_PATTERN = r"^(annotations.[^$]+)$" +URL_PATTERN = regex.compile( + r"^(https?)://" + r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" + r"localhost|" + r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|" + r"\[?[A-F0-9]*:[A-F0-9:]+\]?)" + r"(?::\d+)?" + r"(?:/?|[/?]\S+)$", + regex.IGNORECASE, +) # This class supports overriding the webhook_url only using annotations from yaml files. @@ -15,7 +26,7 @@ # The regex used for label validation is '(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?'. class MsTeamsWebhookUrlTransformer(BaseChannelTransformer): @classmethod - def validate_webhook_override(cls, v: Optional[str]): + def validate_webhook_override(cls, v: Optional[str]) -> Optional[str]: if v: if regex.match(ANNOTATIONS_ONLY_VALUE_PATTERN, v): return "$" + v @@ -24,6 +35,16 @@ def validate_webhook_override(cls, v: Optional[str]): raise ValueError(err_msg) return v + @classmethod + def validate_url_or_get_env(cls, webhook_url: str, default_webhook_url: str) -> str: + if URL_PATTERN.match(webhook_url): + return webhook_url + env_value = os.getenv(webhook_url) + if env_value: + return env_value + + return default_webhook_url + @classmethod def template( cls, @@ -37,5 +58,8 @@ def template( webhook_url = webhook_override webhook_url = cls.process_template_annotations(webhook_url, annotations) + if MISSING in webhook_url: + return default_webhook_url + webhook_url = cls.validate_url_or_get_env(webhook_url, default_webhook_url) - return webhook_url if MISSING not in webhook_url else default_webhook_url + return webhook_url diff --git a/tests/test_ms_teams_transformer.py b/tests/test_ms_teams_transformer.py index 0376b0600..30c6caa70 100644 --- a/tests/test_ms_teams_transformer.py +++ b/tests/test_ms_teams_transformer.py @@ -1,110 +1,165 @@ -import logging +from unittest.mock import patch import pytest from robusta.core.sinks.msteams.msteams_webhook_tranformer import MsTeamsWebhookUrlTransformer -OVERRIDE_WEBHOOK = "override-webhook-url" -DEFAULT_WEBHOOK = "default-webhook-url" - - -testdata_template = [ - ( - "annotations.msteams", - {"msteams": OVERRIDE_WEBHOOK}, - OVERRIDE_WEBHOOK, - "override channel not found", - ), - ( - "annotations.msteams", - {"msteam": OVERRIDE_WEBHOOK}, - DEFAULT_WEBHOOK, - "override - default channel not found", - ), - ( - "$annotations.msteams", - {"msteams": OVERRIDE_WEBHOOK}, - OVERRIDE_WEBHOOK, - "override - default channel not chosen", - ), - ( - "$annotations.msteams", - {"variable": OVERRIDE_WEBHOOK}, - DEFAULT_WEBHOOK, - "override - default channel not chosen", - ), - ( - "${annotations.kubernetes.io/service-name}", - {"kubernetes.io/service-name": OVERRIDE_WEBHOOK}, - OVERRIDE_WEBHOOK, - "override channel not found", - ), - ( - "${annotations.kubernetes.io/service-name}", - {"kubernetes.io/service": OVERRIDE_WEBHOOK}, - DEFAULT_WEBHOOK, - "override - default channel not chosen", - ), - ( - "${annotations.kubernetes.io/service-name}", - {}, - DEFAULT_WEBHOOK, - "override - default channel not chosen", - ), - # webhook_override: "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}" - ( - "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", - {"kubernetes.io/service-name": "yyy"}, - DEFAULT_WEBHOOK, - "override channel not found", - ), - ( - "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", - {"kubernetes.io/service-name": "yyy"}, - DEFAULT_WEBHOOK, - "override - default channel not chosen", - ), +DEFAULT_WEBHOOK_URL = "http://example-default-webhook.com" +OVERRIDE_WEBHOOK_URL = "http://example.com" +OVERRIDE_CONTAINING_ENV_NAME = "WEBHOOK_VALUE" +ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE = None +ENV_MOCK_WITH_WEBHOOK_URL = "http://from-env-example.com" +VALID_URLS_LIST = [ + "http://example.com", + "https://example.com", + "http://127.0.0.1", + "http://127.0.0.1:8080", + "http://example.com/path?query=string", + "https://example.com:443/path/to/resource?query=param#fragment", +] +INVALID_URLS_LIST = [ + "example.com", + "ftp://example.com", + "http://example", + "http://example.com/path with spaces", + "http://-example.com", ] -@pytest.mark.parametrize("webhook_override, annotations, expected, error", testdata_template) -def test_ms_teams_template(webhook_override, annotations, expected, error): - logging.info(f"testing {webhook_override}") - webhook_url = MsTeamsWebhookUrlTransformer.template( - webhook_override=webhook_override, - default_webhook_url=DEFAULT_WEBHOOK, - annotations=annotations, - ) - assert webhook_url == expected, f"{webhook_override} {error}" +@pytest.mark.parametrize( + "webhook_override, annotations, expected, error, env_value", + [ + ( + "annotations.msteams", + {"msteams": OVERRIDE_WEBHOOK_URL}, + OVERRIDE_WEBHOOK_URL, + "override channel not found", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "annotations.msteams", + {"msteams": OVERRIDE_CONTAINING_ENV_NAME}, + ENV_MOCK_WITH_WEBHOOK_URL, + "env with webhook value is not found", + ENV_MOCK_WITH_WEBHOOK_URL, + ), + ( + "annotations.msteams", + {"msteam": OVERRIDE_WEBHOOK_URL}, + DEFAULT_WEBHOOK_URL, + "override - default channel not found", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "$annotations.msteams", + {"msteams": OVERRIDE_WEBHOOK_URL}, + OVERRIDE_WEBHOOK_URL, + "override - default channel not chosen", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "$annotations.msteams", + {"variable": OVERRIDE_WEBHOOK_URL}, + DEFAULT_WEBHOOK_URL, + "override - default channel not chosen", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": OVERRIDE_WEBHOOK_URL}, + OVERRIDE_WEBHOOK_URL, + "override channel not found", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service": OVERRIDE_WEBHOOK_URL}, + DEFAULT_WEBHOOK_URL, + "override - default channel not chosen", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "${annotations.kubernetes.io/service-name}", + {}, + DEFAULT_WEBHOOK_URL, + "override - default channel not chosen", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": "yyy"}, + DEFAULT_WEBHOOK_URL, + "override channel not found", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ( + "$cluster_name-alerts-$annotations.env-${annotations.kubernetes.io/service-name}", + {"kubernetes.io/service-name": "yyy"}, + DEFAULT_WEBHOOK_URL, + "override - default channel not chosen", + ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, + ), + ], +) +def test_ms_teams_webhook_transformer_template_method(webhook_override, annotations, expected, error, env_value): + with patch("robusta.core.sinks.msteams.msteams_webhook_tranformer.os.getenv", return_value=env_value): + webhook_url = MsTeamsWebhookUrlTransformer.template( + webhook_override=webhook_override, + default_webhook_url=DEFAULT_WEBHOOK_URL, + annotations=annotations, + ) + assert webhook_url == expected, f"{webhook_override} {error}" -testdata_validate = [ - ("annotations.team", "$annotations.team", "missing '$' prefix"), - ("$annotations.team", "$annotations.team", "override should be left unchanged"), - ("${annotations.team}", "${annotations.team}", "override should be left unchanged"), -] +@pytest.mark.parametrize( + "webhook_override, env_value, expected, error", + [ + (VALID_URLS_LIST[0], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[0], "webhook url is not valid"), + (VALID_URLS_LIST[1], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[1], "webhook url is not valid"), + (VALID_URLS_LIST[2], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[2], "webhook url is not valid"), + (VALID_URLS_LIST[3], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[3], "webhook url is not valid"), + (VALID_URLS_LIST[4], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[4], "webhook url is not valid"), + (VALID_URLS_LIST[5], ENV_MOCK_WITH_EMPTY_WEBHOOK_VALUE, VALID_URLS_LIST[5], "webhook url is not valid"), + (INVALID_URLS_LIST[0], ENV_MOCK_WITH_WEBHOOK_URL, ENV_MOCK_WITH_WEBHOOK_URL, "webhook url is not valid"), + (INVALID_URLS_LIST[1], ENV_MOCK_WITH_WEBHOOK_URL, ENV_MOCK_WITH_WEBHOOK_URL, "webhook url is not valid"), + (INVALID_URLS_LIST[2], ENV_MOCK_WITH_WEBHOOK_URL, ENV_MOCK_WITH_WEBHOOK_URL, "webhook url is not valid"), + (INVALID_URLS_LIST[3], ENV_MOCK_WITH_WEBHOOK_URL, ENV_MOCK_WITH_WEBHOOK_URL, "webhook url is not valid"), + (INVALID_URLS_LIST[4], ENV_MOCK_WITH_WEBHOOK_URL, ENV_MOCK_WITH_WEBHOOK_URL, "webhook url is not valid"), + ], +) +def test_ms_teams_webhook_transformer_validate_url_or_get_env_method(webhook_override, expected, env_value, error): + with patch("robusta.core.sinks.msteams.msteams_webhook_tranformer.os.getenv", return_value=env_value): + webhook_url = MsTeamsWebhookUrlTransformer.validate_url_or_get_env( + webhook_url=webhook_override, default_webhook_url=DEFAULT_WEBHOOK_URL + ) + assert webhook_url == expected, f"{webhook_override} {error}" -@pytest.mark.parametrize("webhook_override, expected, error", testdata_validate) -def test_ms_teams_validate_webhook_override(webhook_override, expected, error): - logging.info(f"testing {webhook_override}") +@pytest.mark.parametrize( + "webhook_override, expected, error", + [ + ("annotations.team", "$annotations.team", "missing '$' prefix"), + ("$annotations.team", "$annotations.team", "override should be left unchanged"), + ("${annotations.team}", "${annotations.team}", "override should be left unchanged"), + ], +) +def test_ms_teams_webhook_transformer_validate_webhook_override_method(webhook_override, expected, error): webhook_url = MsTeamsWebhookUrlTransformer.validate_webhook_override(webhook_override) assert webhook_url == expected, f"{webhook_override} {error}" -testdata_should_throw = [ - "cluste_name", - "annotations.", - "$annotations.", - "invalid.something", - "labels.", - "$labels.", - "test", -] - - -@pytest.mark.parametrize("webhook_override", testdata_should_throw) -def test_ms_teams_validate_webhook_override_should_throw(webhook_override): - logging.info(f"testing {webhook_override}") +@pytest.mark.parametrize( + "webhook_override", + [ + "cluste_name", + "annotations.", + "$annotations.", + "invalid.something", + "labels.", + "$labels.", + "test", + ], +) +def test_ms_teams_webhook_transformer_validate_webhook_override_raises_error(webhook_override): with pytest.raises(ValueError): MsTeamsWebhookUrlTransformer.validate_webhook_override(webhook_override) From 33c793b1766322df19b85db61d6fcde5611471c5 Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Tue, 2 Jul 2024 17:08:27 +0200 Subject: [PATCH 5/9] * Added docs on how to use webhook_override attribute --- docs/configuration/sinks/ms-teams.rst | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/configuration/sinks/ms-teams.rst b/docs/configuration/sinks/ms-teams.rst index 133961a97..307f246cc 100644 --- a/docs/configuration/sinks/ms-teams.rst +++ b/docs/configuration/sinks/ms-teams.rst @@ -20,6 +20,7 @@ Configuring the MS Teams sink - ms_teams_sink: name: main_ms_teams_sink webhook_url: teams-incoming-webhook # see instructions below + webhook_override: DYNAMIC MS TEAMS WEBHOOK URL OVERRIDE (Optional) Then do a :ref:`Helm Upgrade `. @@ -35,3 +36,35 @@ Obtaining a webhook URL .. image:: /images/msteams_sink/msteam_get_webhook_url.gif :width: 1024 :align: center + + +Dynamic MS Teams Webhook Override +------------------------------------------------------------------- + +You can set the ``MS Teams`` webhook url value dynamically, based on the value of a specific ``annotation`` and environmental variable passed to runner. + +This can be done using the optional ``webhook_override`` sink parameter. + +As for now we support only getting values for annotations, the allowed values for this parameter are: + +- ``annotations.anno`` - The ``MS Teams`` webhook URL will be taken from an annotation with the key anno. +If no such annotation exists, the default webhook will be used. If the annotation is found but its value +does not contain a valid URL, the system will search for an environmental variable with the name of the value + in the ``additional_env_vars`` section of your ``generated_values.yaml`` file. + +For example: + +.. code-block:: yaml + + sinksConfig: + # MS Teams integration params + - ms_teams_sink: + name: main_ms_teams_sink + webhook_url: teams-incoming-webhook # see instructions below + webhook_override: DYNAMIC MS TEAMS WEBHOOK URL OVERRIDE (Optional) + +A replacement pattern is also allowed, using ``$`` sign, before the variable. +For cases where labels or annotations include special characters, such as ``${annotations.kubernetes.io/service-name}``, you can use the `${}` replacement pattern to represent the entire key, including special characters. +For example, if you want to dynamically set the MS Teams webhook url based on the annotation ``kubernetes.io/service-name``, you can use the following syntax: + +- ``webhook_override: "${annotations.kubernetes.io/service-name}"`` From 86b4d8525bf86197ea9e970cff9b6725fcc92f2e Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Wed, 3 Jul 2024 14:38:22 +0200 Subject: [PATCH 6/9] * Updated the docs section mentioning webhook overriding --- docs/configuration/sinks/ms-teams.rst | 25 +++++++++++++++---------- poetry.toml | 2 ++ 2 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 poetry.toml diff --git a/docs/configuration/sinks/ms-teams.rst b/docs/configuration/sinks/ms-teams.rst index 307f246cc..d50f7a351 100644 --- a/docs/configuration/sinks/ms-teams.rst +++ b/docs/configuration/sinks/ms-teams.rst @@ -38,21 +38,16 @@ Obtaining a webhook URL :align: center -Dynamic MS Teams Webhook Override +Dynamically Route MS Teams Alerts ------------------------------------------------------------------- -You can set the ``MS Teams`` webhook url value dynamically, based on the value of a specific ``annotation`` and environmental variable passed to runner. +You can set the MS Teams webhook url value dynamically, based on the value of a specific ``annotation`` and environmental variable passed to runner. This can be done using the optional ``webhook_override`` sink parameter. -As for now we support only getting values for annotations, the allowed values for this parameter are: +As for now, the ``webhook_override`` parameter supports retrieving values specifically from annotations. You can specify an annotation key to retrieve the MS Teams webhook URL using the format ``annotations.``. For example, if you use ``annotations.ms-team-alerts-sink``, the webhook URL will be taken from an annotation with the key ``ms-team-alerts-sink``. -- ``annotations.anno`` - The ``MS Teams`` webhook URL will be taken from an annotation with the key anno. -If no such annotation exists, the default webhook will be used. If the annotation is found but its value -does not contain a valid URL, the system will search for an environmental variable with the name of the value - in the ``additional_env_vars`` section of your ``generated_values.yaml`` file. - -For example: +If the specified annotation does not exist, the default webhook URL from the ``webhook_url`` parameter will be used. If the annotation exists but contains an invalid URL, the system will look for an environmental variable with the name matching the annotation's value. .. code-block:: yaml @@ -61,10 +56,20 @@ For example: - ms_teams_sink: name: main_ms_teams_sink webhook_url: teams-incoming-webhook # see instructions below - webhook_override: DYNAMIC MS TEAMS WEBHOOK URL OVERRIDE (Optional) + webhook_override: "annotations.ms-team-alerts-sink" A replacement pattern is also allowed, using ``$`` sign, before the variable. For cases where labels or annotations include special characters, such as ``${annotations.kubernetes.io/service-name}``, you can use the `${}` replacement pattern to represent the entire key, including special characters. For example, if you want to dynamically set the MS Teams webhook url based on the annotation ``kubernetes.io/service-name``, you can use the following syntax: - ``webhook_override: "${annotations.kubernetes.io/service-name}"`` + +Example: + +.. code-block:: yaml + + sinksConfig: + - ms_teams_sink: + name: main_ms_teams_sink + webhook_url: teams-incoming-webhook # see instructions below + webhook_override: ${annotations.kubernetes.io/service-name} diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 000000000..3b549d6c8 --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +create = true From 09f0dda33b6a3aab55eef03cdd991670c1091060 Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Wed, 3 Jul 2024 14:39:23 +0200 Subject: [PATCH 7/9] * Removed redundant poetry virtualenv file --- poetry.toml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 poetry.toml diff --git a/poetry.toml b/poetry.toml deleted file mode 100644 index 3b549d6c8..000000000 --- a/poetry.toml +++ /dev/null @@ -1,2 +0,0 @@ -[virtualenvs] -create = true From 82901a01e4aaacebe0384c2b15bfa30fc61bc69e Mon Sep 17 00:00:00 2001 From: Dmitry Chevtaev Date: Wed, 3 Jul 2024 15:01:26 +0200 Subject: [PATCH 8/9] * Updated Dynamically Route MS Teams Alerts docs section --- docs/configuration/sinks/ms-teams.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/sinks/ms-teams.rst b/docs/configuration/sinks/ms-teams.rst index d50f7a351..52b10623f 100644 --- a/docs/configuration/sinks/ms-teams.rst +++ b/docs/configuration/sinks/ms-teams.rst @@ -47,7 +47,7 @@ This can be done using the optional ``webhook_override`` sink parameter. As for now, the ``webhook_override`` parameter supports retrieving values specifically from annotations. You can specify an annotation key to retrieve the MS Teams webhook URL using the format ``annotations.``. For example, if you use ``annotations.ms-team-alerts-sink``, the webhook URL will be taken from an annotation with the key ``ms-team-alerts-sink``. -If the specified annotation does not exist, the default webhook URL from the ``webhook_url`` parameter will be used. If the annotation exists but contains an invalid URL, the system will look for an environmental variable with the name matching the annotation's value. +If the specified annotation does not exist, the default webhook URL from the ``webhook_url`` parameter will be used. If the annotation exists but does not contain a URL, the system will look for an environmental variable with the name matching the ``annotation`` value. .. code-block:: yaml From 3a9f117fd383eb965c436008e107be766a93c498 Mon Sep 17 00:00:00 2001 From: Dima Chievtaiev Date: Sun, 7 Jul 2024 12:13:03 +0200 Subject: [PATCH 9/9] *Added logging to MsTeamsWebhookUrlTransformer's validate_url_or_get_env method --- src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py index 851756118..9c467a2d8 100644 --- a/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py +++ b/src/robusta/core/sinks/msteams/msteams_webhook_tranformer.py @@ -1,3 +1,4 @@ +import logging import os from typing import Dict, Optional @@ -39,9 +40,12 @@ def validate_webhook_override(cls, v: Optional[str]) -> Optional[str]: def validate_url_or_get_env(cls, webhook_url: str, default_webhook_url: str) -> str: if URL_PATTERN.match(webhook_url): return webhook_url + logging.info(f"URL matching failed for: {webhook_url}. Trying to get environment variable.") + env_value = os.getenv(webhook_url) if env_value: return env_value + logging.info(f"Environment variable not found for: {webhook_url}. Using default webhook URL.") return default_webhook_url