From 1a7dc4840104b908a4f7082de110d9b067fcb564 Mon Sep 17 00:00:00 2001
From: Ewoud Kohl van Wijngaarden <ewoud@kohlvanwijngaarden.nl>
Date: Wed, 27 Mar 2024 20:26:12 +0100
Subject: [PATCH] Make browser log level threshold configurable

Previously it was tuned to only trigger on SEVERE messages, but the
desire is to make it stricter and also fail tests when there are
warnings. By making it a variable, it can be changed without changing
the code.
---
 tests/conftest.py   |  4 ++++
 tests/test_pages.py | 27 ++++++++++++++++++++++++---
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 70d010e..de5f7c7 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -8,6 +8,10 @@
 User = namedtuple('User', ['username', 'password', 'name'])
 
 
+def pytest_configure(config):
+    config.addinivalue_line('markers', 'internal: mark test as a self test')
+
+
 def pytest_generate_tests(metafunc):
     variables = metafunc.config._variables  # pylint: disable=protected-access
 
diff --git a/tests/test_pages.py b/tests/test_pages.py
index 6b23e0c..db25d2a 100644
--- a/tests/test_pages.py
+++ b/tests/test_pages.py
@@ -1,6 +1,7 @@
 # pylint: disable=redefined-outer-name
 from __future__ import print_function
 
+from enum import IntFlag
 from typing import Union, KeysView, List
 from urllib.parse import ParseResult, urlparse, parse_qs, urlencode
 
@@ -12,6 +13,10 @@
 from selenium.common.exceptions import TimeoutException
 
 
+# https://github.com/SeleniumHQ/selenium/wiki/Logging
+BrowserLogLevel = IntFlag('BrowserLogLevel', ['ALL', 'DEBUG', 'INFO', 'WARNING', 'SEVERE', 'OFF'])
+
+
 EXCLUDE_ERRORS = (
     # Fixed in Foreman 3.7 - https://projects.theforeman.org/issues/36093
     'Scrollbar test exception: TypeError:',
@@ -52,7 +57,7 @@ def filtered_url_query(url: str, allowed_query_params: Union[List, KeysView]) ->
 
 @pytest.mark.nondestructive
 @pytest.mark.selenium
-def test_menu_item(selenium, user, url):
+def test_menu_item(selenium, user, url, variables):
     selenium.get(url)
     assert selenium.current_url.endswith('/users/login'), 'Redirect to login page'
     login_field = selenium.find_element(By.NAME, 'login[login]')
@@ -82,5 +87,21 @@ def test_menu_item(selenium, user, url):
         print("https://github.com/mozilla/geckodriver/issues/284")
 
     logs = selenium.get_log('browser')
-    severe_messages = [x['message'] for x in logs if x.get('level') == 'SEVERE' and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
-    assert severe_messages == [], 'Error messages with log level SEVERE in browser console'
+    threshold = BrowserLogLevel[variables.get('browser_log_threshold', 'SEVERE')]
+    messages = [x['message'] for x in logs
+                if BrowserLogLevel[x['level']] >= threshold
+                and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
+    assert messages == [], f'Messages with log level {threshold} or above in browser console'
+
+@pytest.mark.parametrize('level,expected',
+                         [
+                             ('ALL', True),
+                             ('SEVERE', True),
+                             ('OFF', False),
+                             ])
+@pytest.mark.internal
+def test_browser_log_level(level, expected):
+    """
+    Self test the logic to determine if browser log matches
+    """
+    assert (BrowserLogLevel.SEVERE >= BrowserLogLevel[level]) == expected