Make TLS E2E tests pass

.github/workflows/pr.yml

@@ -33,22 +33,38 @@ jobs:
       zip_url: 'https://agent-ints-python-build-sandbox.s3.eu-north-1.amazonaws.com/python-windows-combined-v3.12.6-openssl-3.0.15-openssl-3.0.9-amd64.zip'
     secrets: inherit
 
-  test-tls-linux:
+  test-tls-linux-inactive:
     uses: ./.github/workflows/test-target.yml
     with:
-      job-name: TLS
+      job-name: TLS-linux-FIPS-inactive
       target: tls
       platform: linux
       runner: '["ubuntu-22.04"]'
       repo: "core"
       standard: true
-      agent-image: "datadog/agent-dev:nicolas-guerguadj-add-fips-mode-py3"
+      pytest-args: "-s -k test_fips_inactive_e2e"
+      fips-mode: false
+      agent-image: "datadog/agent-dev:main-fips"
+    secrets: inherit
+
+  test-tls-linux-active:
+    uses: ./.github/workflows/test-target.yml
+    with:
+      job-name: TLS-linux-FIPS-active
+      target: tls
+      platform: linux
+      runner: '["ubuntu-22.04"]'
+      repo: "core"
+      standard: true
+      pytest-args: "-s -k test_fips_active_e2e"
+      fips-mode: true
+      agent-image: "datadog/agent-dev:main-fips"
     secrets: inherit
 
   test-tls-windows:
     uses: ./.github/workflows/test-target.yml
     with:
-      job-name: TLS
+      job-name: TLS-windows
       target: tls
       platform: windows
       runner: '["windows-2022"]'

.github/workflows/test-target.yml

@@ -85,6 +85,11 @@ on:
         required: false
         type: string
         default: ""
+      fips-mode:
+        description: "Should the agent be run in FIPS mode for e2e tests"
+        required: false
+        type: boolean
+        default: false
 
 defaults:
   run:
@@ -270,7 +275,7 @@ jobs:
         #   by default
         if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
           set +e # Disable immediate exit
-          ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -281,7 +286,7 @@ jobs:
           fi
         elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
           set +e # Disable immediate exit
-          ddev env test --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -291,7 +296,7 @@ jobs:
             exit $exit_code
           fi
         else
-          ddev env test --base --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
         fi
 
     - name: Run E2E tests
@@ -308,7 +313,7 @@ jobs:
         #   by default
         if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
           set +e # Disable immediate exit
-          ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -319,7 +324,7 @@ jobs:
           fi
         elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
           set +e # Disable immediate exit
-          ddev env test --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -329,7 +334,7 @@ jobs:
             exit $exit_code
           fi
         else
-          ddev env test --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --new-env --junit ${{ inputs.target }} ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
         fi
 
     - name: Run benchmarks
@@ -355,7 +360,7 @@ jobs:
         #   by default
         if [ '${{ inputs.pytest-args }}' = '-m flaky' ]; then
           set +e # Disable immediate exit
-          ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -366,7 +371,7 @@ jobs:
           fi
         elif [ '${{ inputs.pytest-args }}' = '-m "not flaky"' ]; then
           set +e # Disable immediate exit
-          ddev env test --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest -- all ${{ inputs.pytest-args }}
           exit_code=$?
           if [ $exit_code -eq 5 ]; then
             # Flaky test count can be zero, this is done to avoid pipeline failure
@@ -376,7 +381,7 @@ jobs:
             exit $exit_code
           fi
         else
-          ddev env test --base --new-env --junit ${{ inputs.target }}:latest ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
+          ddev env test -e GOFIPS=${{ inputs.fips-mode && '1' || '0' }} --base --new-env --junit ${{ inputs.target }}:latest ${{ inputs.pytest-args != '' && format('-- all {0}', inputs.pytest-args) || '' }}
         fi
 
     - name: View trace log

datadog_checks_base/datadog_checks/base/checks/base.py

@@ -6,6 +6,7 @@
 import importlib
 import inspect
 import logging
+import os
 import re
 import traceback
 import unicodedata
@@ -46,6 +47,7 @@
 from ..utils.agent.utils import should_profile_memory
 from ..utils.common import ensure_bytes, to_native_string
 from ..utils.diagnose import Diagnosis
+from ..utils.fips import enable_fips
 from ..utils.http import RequestsWrapper
 from ..utils.limiter import Limiter
 from ..utils.metadata import MetadataManager
@@ -307,6 +309,40 @@ def __init__(self, *args, **kwargs):
         self.__formatted_tags = None
         self.__logs_enabled = None
 
+        if os.environ.get("GOFIPS", "0") == "1" or self.instance.get("fips", False):
+            with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "r") as f:
+                print(f.read())
+            with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "w") as f:
+                config = """
+config_diagnostics = 1
+openssl_conf = openssl_init
+
+.include /opt/datadog-agent/embedded/ssl/fipsmodule.cnf
+
+[openssl_init]
+providers = provider_sect
+alg_section = algorithm_sect
+
+[provider_sect]
+fips = fips_sect
+base = base_sect
+
+[base_sect]
+activate = 1
+
+[algorithm_sect]
+default_properties = fips=yes
+"""
+                f.write(config)
+                print("wrote")
+            with open("/opt/datadog-agent/embedded/ssl/openssl.cnf", "r") as f:
+                print(f.read())
+
+            enable_fips(
+                path_to_openssl_conf="/opt/datadog-agent/embedded/ssl/openssl.cnf",
+                path_to_openssl_modules="/opt/datadog-agent/embedded/lib/ossl-modules",
+            )
+
     def _create_metrics_pattern(self, metric_patterns, option_name):
         all_patterns = metric_patterns.get(option_name, [])
 

ddev/src/ddev/e2e/agent/docker.py

@@ -113,7 +113,12 @@ def start(self, *, agent_build: str, local_packages: dict[Path, str], env_vars:
 
         if agent_build.startswith("datadog/"):
             # Add a potentially missing `py` suffix for default non-RC builds
-            if 'rc' not in agent_build and 'py' not in agent_build and not re.match(AGENT_VERSION_REGEX, agent_build):
+            if (
+                'rc' not in agent_build
+                and 'py' not in agent_build
+                and 'fips' not in agent_build
+                and not re.match(AGENT_VERSION_REGEX, agent_build)
+            ):
                 agent_build = f'{agent_build}-py{self.python_version[0]}'
 
             if self.metadata.get('use_jmx') and not agent_build.endswith('-jmx'):

tls/tests/conftest.py

@@ -29,6 +29,14 @@
 }
 
 
+@pytest.fixture(scope="function")
+def clean_fips_environment():
+    os.environ["GOFIPS"] = "0"
+    os.environ["OPENSSL_CONF"] = ""
+    os.environ["OPENSSL_MODULES"] = ""
+    yield
+
+
 @pytest.fixture(scope='session')
 def dd_environment(instance_e2e, mock_local_tls_dns):
     with docker_run(os.path.join(HERE, 'compose', 'docker-compose.yml'), build=True, sleep=20):
@@ -173,6 +181,7 @@ def instance_e2e_fips():
         'tls_ca_cert': CA_CERT_MOUNT_PATH,
         'tls_verify': False,
         'tls_validate_hostname': False,
+        'fips': True,  # TODO: remove in favor of env var GOFIPS
     }
 
 

tls/tests/fips/Dockerfile

@@ -9,7 +9,3 @@ RUN chmod +x /usr/local/bin/start-server.sh
 
 # Expose port 443
 EXPOSE 443
-
-# Run the server script
-CMD ["/usr/local/bin/start-server.sh"]
-

tls/tests/fips/start-server.sh

@@ -17,4 +17,5 @@ openssl s_server \
     -cert /etc/ssl/certs/server.crt \
     -key /etc/ssl/private/server.key \
     -cipher $CIPHER \
+    -no_tls1_3 \
     -WWW

tls/tests/test_fips_active_e2e.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import os
+from typing import Any  # noqa: F401
+
+import pytest
+
+from datadog_checks.tls import TLSCheck
+from datadog_checks.tls.const import (
+    SERVICE_CHECK_CAN_CONNECT,
+    SERVICE_CHECK_VALIDATION,
+)
+
+
+@pytest.mark.e2e
+def test_connection_after_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips):
+    """
+    Test connection to the FIPS server after enabling FIPS mode.
+    """
+    os.environ["GOFIPS"] = "1"
+    aggregator = dd_agent_check(instance_e2e_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)
+
+
+@pytest.mark.e2e
+def test_connection_after_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips):
+    """
+    Test connection to the non-FIPS server after enabling FIPS mode.
+    """
+    os.environ["GOFIPS"] = "1"
+    aggregator = dd_agent_check(instance_e2e_non_fips)
+    aggregator.assert_service_check(
+        SERVICE_CHECK_VALIDATION,
+        message="[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] ssl/tls alert handshake failure (_ssl.c:1000)",
+    )

tls/tests/test_fips_inactive_e2e.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from typing import Any  # noqa: F401
+
+import pytest
+
+from datadog_checks.tls import TLSCheck
+from datadog_checks.tls.const import (
+    SERVICE_CHECK_CAN_CONNECT,
+    SERVICE_CHECK_VALIDATION,
+)
+
+
+@pytest.mark.e2e
+def test_connection_before_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips):
+    """
+    Test connection to the FIPS server before enabling FIPS mode.
+    """
+    aggregator = dd_agent_check(instance_e2e_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)
+
+
+@pytest.mark.e2e
+def test_connection_before_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips):
+    """
+    Test connection to the non-FIPS server before enabling FIPS mode.
+    """
+    aggregator = dd_agent_check(instance_e2e_non_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)