From 520a571eace56e3b12787448216414ba47a7682d Mon Sep 17 00:00:00 2001 From: David Kirov Date: Tue, 10 Dec 2024 16:59:58 +0100 Subject: [PATCH] Make TLS E2E tests pass --- .github/workflows/pr.yml | 24 +++++- .github/workflows/test-target.yml | 29 ++++--- .../datadog_checks/base/checks/base.py | 36 +++++++++ ddev/src/ddev/e2e/agent/docker.py | 7 +- tls/tests/conftest.py | 9 +++ tls/tests/fips/Dockerfile | 4 - tls/tests/fips/start-server.sh | 1 + tls/tests/test_fips_active_e2e.py | 36 +++++++++ tls/tests/test_fips_e2e.py | 76 ------------------- tls/tests/test_fips_inactive_e2e.py | 34 +++++++++ 10 files changed, 163 insertions(+), 93 deletions(-) create mode 100644 tls/tests/test_fips_active_e2e.py delete mode 100644 tls/tests/test_fips_e2e.py create mode 100644 tls/tests/test_fips_inactive_e2e.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0a65113611bf7..618c96527dc95 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -33,7 +33,7 @@ jobs: zip_url: 'https://agent-ints-python-build-sandbox.s3.eu-north-1.amazonaws.com/python-windows-combined-v3.12.6-openssl-3.0.15-openssl-3.0.9-amd64.zip' secrets: inherit - test-tls-linux: + test-tls-linux-inactive: uses: ./.github/workflows/test-target.yml with: job-name: TLS @@ -42,7 +42,27 @@ jobs: runner: '["ubuntu-22.04"]' repo: "core" standard: true - agent-image: "datadog/agent-dev:nicolas-guerguadj-add-fips-mode-py3" + pytest-args: "-s -k test_fips_inactive_e2e" + fips-mode: false + traces-artifact-name: traces-tls-linux-no-fips + test-results-name: test-tls-linux-no-fips + agent-image: "datadog/agent-dev:main-fips" + secrets: inherit + + test-tls-linux-active: + uses: ./.github/workflows/test-target.yml + with: + job-name: TLS + target: tls + platform: linux + runner: '["ubuntu-22.04"]' + repo: "core" + standard: true + pytest-args: "-s -k test_fips_active_e2e" + fips-mode: true + traces-artifact-name: traces-tls-linux-fips + test-results-name: test-tls-linux-fips + agent-image: "datadog/agent-dev:main-fips" secrets: inherit test-tls-windows: diff --git a/.github/workflows/test-target.yml b/.github/workflows/test-target.yml index 489620e36dd94..175465dec45a5 100644 --- a/.github/workflows/test-target.yml +++ b/.github/workflows/test-target.yml @@ -72,6 +72,10 @@ on: required: false default: "traces" type: string + test-results-name: + required: false + default: "test-results" + type: string trace-agent-port: required: false default: "8126" @@ -85,6 +89,11 @@ on: required: false type: string default: "" + fips-mode: + description: "Should the agent be run in FIPS mode for e2e tests" + required: false + type: boolean + default: false defaults: run: @@ -270,7 +279,7 @@ jobs: # by default if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then set +e # Disable immediate exit - ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -281,7 +290,7 @@ jobs: fi elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then set +e # Disable immediate exit - ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -291,7 +300,7 @@ jobs: exit $exit_code fi else - ddev env test --base --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} fi - name: Run E2E tests @@ -308,7 +317,7 @@ jobs: # by default if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then set +e # Disable immediate exit - ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -319,7 +328,7 @@ jobs: fi elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then set +e # Disable immediate exit - ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -329,7 +338,7 @@ jobs: exit $exit_code fi else - ddev env test --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} fi - name: Run benchmarks @@ -355,7 +364,7 @@ jobs: # by default if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then set +e # Disable immediate exit - ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -366,7 +375,7 @@ jobs: fi elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then set +e # Disable immediate exit - ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }} exit_code=$? if [ $exit_code -eq 5 ]; then # Flaky test count can be zero, this is done to avoid pipeline failure @@ -376,7 +385,7 @@ jobs: exit $exit_code fi else - ddev env test --base --new-env --junit ${{ inputs.target }}:latest ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} + ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }} fi - name: View trace log @@ -402,7 +411,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: "test-results-${{ inputs.target }}-${{ inputs.platform }}" + name: "${{ inputs.test-results-name }}-${{ inputs.target }}-${{ inputs.platform }}" path: "${{ env.TEST_RESULTS_BASE_DIR }}" - name: Upload coverage data diff --git a/datadog_checks_base/datadog_checks/base/checks/base.py b/datadog_checks_base/datadog_checks/base/checks/base.py index c87b2cdb70479..046a2c2cb16f2 100644 --- a/datadog_checks_base/datadog_checks/base/checks/base.py +++ b/datadog_checks_base/datadog_checks/base/checks/base.py @@ -6,6 +6,7 @@ import importlib import inspect import logging +import os import re import traceback import unicodedata @@ -46,6 +47,7 @@ from ..utils.agent.utils import should_profile_memory from ..utils.common import ensure_bytes, to_native_string from ..utils.diagnose import Diagnosis +from ..utils.fips import enable_fips from ..utils.http import RequestsWrapper from ..utils.limiter import Limiter from ..utils.metadata import MetadataManager @@ -307,6 +309,40 @@ def __init__(self, *args, **kwargs): self.__formatted_tags = None self.__logs_enabled = None + if os.environ.get("GOFIPS", "0") == "1" or self.instance.get("fips", False): + with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "r") as f: + print(f.read()) + with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "w") as f: + config = """ +config_diagnostics = 1 +openssl_conf = openssl_init + +.include /opt/datadog-agent/embedded/ssl/fipsmodule.cnf + +[openssl_init] +providers = provider_sect +alg_section = algorithm_sect + +[provider_sect] +fips = fips_sect +base = base_sect + +[base_sect] +activate = 1 + +[algorithm_sect] +default_properties = fips=yes +""" + f.write(config) + print("wrote") + with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "r") as f: + print(f.read()) + + enable_fips( + path_to_openssl_conf="/opt/datadog-agent/embedded/ssl/openssl.cnf", + path_to_openssl_modules="/opt/datadog-agent/embedded/lib/ossl-modules", + ) + def _create_metrics_pattern(self, metric_patterns, option_name): all_patterns = metric_patterns.get(option_name, []) diff --git a/ddev/src/ddev/e2e/agent/docker.py b/ddev/src/ddev/e2e/agent/docker.py index 1821d50d1b346..206d4e2bbc1ee 100644 --- a/ddev/src/ddev/e2e/agent/docker.py +++ b/ddev/src/ddev/e2e/agent/docker.py @@ -113,7 +113,12 @@ def start(self, *, agent_build: str, local_packages: dict[Path, str], env_vars: if agent_build.startswith("datadog/"): # Add a potentially missing `py` suffix for default non-RC builds - if 'rc' not in agent_build and 'py' not in agent_build and not re.match(AGENT_VERSION_REGEX, agent_build): + if ( + 'rc' not in agent_build + and 'py' not in agent_build + and 'fips' not in agent_build + and not re.match(AGENT_VERSION_REGEX, agent_build) + ): agent_build = f'{agent_build}-py{self.python_version[0]}' if self.metadata.get('use_jmx') and not agent_build.endswith('-jmx'): diff --git a/tls/tests/conftest.py b/tls/tests/conftest.py index 3b765be4b4071..49018f9e3cc11 100644 --- a/tls/tests/conftest.py +++ b/tls/tests/conftest.py @@ -29,6 +29,14 @@ } +@pytest.fixture(scope="function") +def clean_fips_environment(): + os.environ["GOFIPS"] = "0" + os.environ["OPENSSL_CONF"] = "" + os.environ["OPENSSL_MODULES"] = "" + yield + + @pytest.fixture(scope='session') def dd_environment(instance_e2e, mock_local_tls_dns): with docker_run(os.path.join(HERE, 'compose', 'docker-compose.yml'), build=True, sleep=20): @@ -173,6 +181,7 @@ def instance_e2e_fips(): 'tls_ca_cert': CA_CERT_MOUNT_PATH, 'tls_verify': False, 'tls_validate_hostname': False, + 'fips': True, # TODO: remove in favor of env var GOFIPS } diff --git a/tls/tests/fips/Dockerfile b/tls/tests/fips/Dockerfile index 1e666418eea4e..dbf827d8f9d66 100644 --- a/tls/tests/fips/Dockerfile +++ b/tls/tests/fips/Dockerfile @@ -9,7 +9,3 @@ RUN chmod +x /usr/local/bin/start-server.sh # Expose port 443 EXPOSE 443 - -# Run the server script -CMD ["/usr/local/bin/start-server.sh"] - diff --git a/tls/tests/fips/start-server.sh b/tls/tests/fips/start-server.sh index 825c24125c8b6..e4eb4155ce18c 100644 --- a/tls/tests/fips/start-server.sh +++ b/tls/tests/fips/start-server.sh @@ -17,4 +17,5 @@ openssl s_server \ -cert /etc/ssl/certs/server.crt \ -key /etc/ssl/private/server.key \ -cipher $CIPHER \ + -no_tls1_3 \ -WWW diff --git a/tls/tests/test_fips_active_e2e.py b/tls/tests/test_fips_active_e2e.py new file mode 100644 index 0000000000000..74a831c3dc785 --- /dev/null +++ b/tls/tests/test_fips_active_e2e.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any # noqa: F401 + +import pytest + +from datadog_checks.tls import TLSCheck +from datadog_checks.tls.const import ( + SERVICE_CHECK_CAN_CONNECT, + SERVICE_CHECK_VALIDATION, +) + + +@pytest.mark.e2e +def test_connection_after_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips): + """ + Test connection to the FIPS server after enabling FIPS mode. + """ + aggregator = dd_agent_check(instance_e2e_fips) + aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) + aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1) + + +@pytest.mark.e2e +def test_connection_after_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips): + """ + Test connection to the non-FIPS server after enabling FIPS mode. + """ + aggregator = dd_agent_check(instance_e2e_non_fips) + aggregator.assert_service_check( + SERVICE_CHECK_VALIDATION, + message="[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] ssl/tls alert handshake failure (_ssl.c:1000)", + ) diff --git a/tls/tests/test_fips_e2e.py b/tls/tests/test_fips_e2e.py deleted file mode 100644 index 7cf79a99577d1..0000000000000 --- a/tls/tests/test_fips_e2e.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding: utf-8 -*- - -# (C) Datadog, Inc. 2024-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import os -from typing import Any # noqa: F401 - -import pytest - -from datadog_checks.tls import TLSCheck -from datadog_checks.tls.const import ( - SERVICE_CHECK_CAN_CONNECT, - SERVICE_CHECK_EXPIRATION, - SERVICE_CHECK_VALIDATION, - SERVICE_CHECK_VERSION, -) - - -@pytest.fixture(scope="function") -def clean_environment(): - os.environ["GOFIPS"] = "0" - os.environ["OPENSSL_CONF"] = "" - os.environ["OPENSSL_MODULES"] = "" - yield - - -@pytest.mark.e2e -def test_connection_before_fips(clean_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips): - """ - Test connection to the FIPS server before enabling FIPS mode. - """ - aggregator = dd_agent_check(instance_e2e_fips) - aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VERSION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_EXPIRATION, status=TLSCheck.OK, count=1) - - - -@pytest.mark.e2e -def test_connection_before_non_fips(clean_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips): - """ - Test connection to the non-FIPS server before enabling FIPS mode. - """ - aggregator = dd_agent_check(instance_e2e_non_fips) - aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VERSION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_EXPIRATION, status=TLSCheck.OK, count=1) - - -@pytest.mark.e2e -def test_connection_after_fips(clean_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips): - """ - Test connection to the FIPS server after enabling FIPS mode. - """ - os.environ["GOFIPS"] = "1" - aggregator = dd_agent_check(instance_e2e_fips) - aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VERSION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1) - aggregator.assert_service_check(SERVICE_CHECK_EXPIRATION, status=TLSCheck.OK, count=1) - - -@pytest.mark.e2e -def test_connection_after_non_fips(clean_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips): - """ - Test connection to the non-FIPS server after enabling FIPS mode. - """ - os.environ["GOFIPS"] = "1" - aggregator = dd_agent_check(instance_e2e_non_fips) - aggregator.assert_service_check( - SERVICE_CHECK_VALIDATION, - message="[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] sslv3 alert handshake failure (_ssl.c:1006)", - ) diff --git a/tls/tests/test_fips_inactive_e2e.py b/tls/tests/test_fips_inactive_e2e.py new file mode 100644 index 0000000000000..9d4a33a822a1c --- /dev/null +++ b/tls/tests/test_fips_inactive_e2e.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +# (C) Datadog, Inc. 2024-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) +from typing import Any # noqa: F401 + +import pytest + +from datadog_checks.tls import TLSCheck +from datadog_checks.tls.const import ( + SERVICE_CHECK_CAN_CONNECT, + SERVICE_CHECK_VALIDATION, +) + + +@pytest.mark.e2e +def test_connection_before_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips): + """ + Test connection to the FIPS server before enabling FIPS mode. + """ + aggregator = dd_agent_check(instance_e2e_fips) + aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) + aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1) + + +@pytest.mark.e2e +def test_connection_before_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips): + """ + Test connection to the non-FIPS server before enabling FIPS mode. + """ + aggregator = dd_agent_check(instance_e2e_non_fips) + aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1) + aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)