Merge e2e tests and clean test-fips workflow

.github/workflows/test-fips.yml

@@ -1,4 +1,4 @@
-name: Test FIPS
+name: Test FIPS E2E
 
 on:
   workflow_dispatch:
@@ -83,9 +83,6 @@ jobs:
         ddev config set repos.core .
         ddev config set repo core
 
-    - name: Lint
-      run: ddev test --lint ${{ inputs.target || 'tls' }}
-
     - name: Prepare for testing
       env:
         PYTHONUNBUFFERED: "1"
@@ -105,41 +102,17 @@ jobs:
         mkdir "${{ env.TRACE_CAPTURE_BASE_DIR }}"
         python .ddev/ci/scripts/traces.py capture --port "8126" --record-file "${{ env.TRACE_CAPTURE_FILE }}" > "${{ env.TRACE_CAPTURE_LOG }}" 2>&1 &
 
-    # - name: Run Unit & Integration tests
-    #   env:
-    #     DDEV_TEST_ENABLE_TRACING: "1"
-    #   run: |
-    #     ddev test --cov --junit ${{ inputs.target || 'tls' }} -- '-k fips'
-
-    # - name: Run Unit & Integration tests with minimum version of base package
-    #   run: |
-    #     ddev test --compat --recreate --junit ${{ inputs.target || 'tls' }} -- '-k fips'
-
-    # - name: Run E2E tests with latest base package
-    #   env:
-    #     DD_API_KEY: "${{ secrets.DD_API_KEY }}"
-    #   run: |
-    #     # '-- all' is passed for e2e tests if pytest args are provided
-    #     # This is done to avoid ddev from interpreting the arguments as environments
-    #     #   instead of pytest-args, because by default if no environment is provided
-    #     #   after -- it will run all environments. So when pytests args are provided
-    #     #   ddev will interpret '-m' as an environment to run the e2e test on and fails
-    #     # This is not required when no pytest args are provided and it will run all environments
-    #     #   by default
-    #     ddev env test -e GOFIPS=1 --base --new-env --junit ${{ inputs.target || 'tls' }} -- all -k fips
-
-    - name: Run E2E tests
+    - name: Run E2E tests with FIPS disabled
+      env:
+        DD_API_KEY: "${{ secrets.DD_API_KEY }}"
+      run: |
+        ddev env test -e GOFIPS=0 --new-env --junit ${{ inputs.target || 'tls' }} -- all -m "fips_off"
+
+    - name: Run E2E tests with FIPS enabled
       env:
         DD_API_KEY: "${{ secrets.DD_API_KEY }}"
       run: |
-        ddev env test -e GOFIPS=0 --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_inactive"
-        ddev env test -e GOFIPS=1 --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_active"
-
-    # - name: Run benchmarks
-    #   run: ddev test --bench --junit ${{ inputs.target || 'tls' }}
-    #
-    # - name: Run tests and verify support for the latest version
-    #   run: ddev test --latest --junit ${{ inputs.target || 'tls' }}
+        ddev env test -e GOFIPS=1 --new-env --junit ${{ inputs.target || 'tls' }} -- all -k "fips_on"
 
     - name: View trace log
       if: always()

tls/tests/test_fips.py

@@ -0,0 +1,58 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+from typing import Any  # noqa: F401
+
+import pytest
+
+from datadog_checks.tls import TLSCheck
+from datadog_checks.tls.const import (
+    SERVICE_CHECK_CAN_CONNECT,
+    SERVICE_CHECK_VALIDATION,
+)
+
+
+@pytest.mark.e2e
+@pytest.mark.fips_off
+def test_connection_before_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips):
+    """
+    Connection to the FIPS server before enabling FIPS mode should succeed.
+    """
+    aggregator = dd_agent_check(instance_e2e_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)
+
+
+@pytest.mark.e2e
+@pytest.mark.fips_off
+def test_connection_before_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips):
+    """
+    Connection to the non-FIPS server before enabling FIPS mode should succeed.
+    """
+    aggregator = dd_agent_check(instance_e2e_non_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)
+
+
+@pytest.mark.e2e
+@pytest.mark.fips_on
+def test_connection_after_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_fips):
+    """
+    Connection to the FIPS server after enabling FIPS mode should succeed.
+    """
+    aggregator = dd_agent_check(instance_e2e_fips)
+    aggregator.assert_service_check(SERVICE_CHECK_CAN_CONNECT, status=TLSCheck.OK, count=1)
+    aggregator.assert_service_check(SERVICE_CHECK_VALIDATION, status=TLSCheck.OK, count=1)
+
+
+@pytest.mark.e2e
+@pytest.mark.fips_on
+def test_connection_after_non_fips(clean_fips_environment, dd_fips_environment, dd_agent_check, instance_e2e_non_fips):
+    """
+    Connection to the non-FIPS server after enabling FIPS mode should fail.
+    """
+    aggregator = dd_agent_check(instance_e2e_non_fips)
+    aggregator.assert_service_check(
+        SERVICE_CHECK_VALIDATION,
+        message="[SSL: SSLV3_ALERT_HANDSHAKE_FAILURE] ssl/tls alert handshake failure (_ssl.c:1000)",
+    )