Make browser log level threshold configurable

tests/conftest.py

@@ -8,6 +8,10 @@
 User = namedtuple('User', ['username', 'password', 'name'])
 
 
+def pytest_configure(config):
+    config.addinivalue_line('markers', 'internal: mark test as a self test')
+
+
 def pytest_generate_tests(metafunc):
     variables = metafunc.config._variables  # pylint: disable=protected-access
 

tests/test_pages.py

@@ -1,6 +1,7 @@
 # pylint: disable=redefined-outer-name
 from __future__ import print_function
 
+from enum import IntFlag
 from typing import Union, KeysView, List
 from urllib.parse import ParseResult, urlparse, parse_qs, urlencode
 
@@ -12,6 +13,10 @@
 from selenium.common.exceptions import TimeoutException
 
 
+# https://github.com/SeleniumHQ/selenium/wiki/Logging
+BrowserLogLevel = IntFlag('BrowserLogLevel', ['ALL', 'DEBUG', 'INFO', 'WARNING', 'SEVERE', 'OFF'])
+
+
 EXCLUDE_ERRORS = (
     # Fixed in Foreman 3.7 - https://projects.theforeman.org/issues/36093
     'Scrollbar test exception: TypeError:',
@@ -52,7 +57,7 @@ def filtered_url_query(url: str, allowed_query_params: Union[List, KeysView]) ->
 
 @pytest.mark.nondestructive
 @pytest.mark.selenium
-def test_menu_item(selenium, user, url):
+def test_menu_item(selenium, user, url, variables):
     selenium.get(url)
     assert selenium.current_url.endswith('/users/login'), 'Redirect to login page'
     login_field = selenium.find_element(By.NAME, 'login[login]')
@@ -82,5 +87,22 @@ def test_menu_item(selenium, user, url):
         print("https://github.com/mozilla/geckodriver/issues/284")
 
     logs = selenium.get_log('browser')
-    severe_messages = [x['message'] for x in logs if x.get('level') == 'SEVERE' and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
-    assert severe_messages == [], 'Error messages with log level SEVERE in browser console'
+    threshold = BrowserLogLevel[variables.get('browser_log_threshold', 'SEVERE')]
+    messages = [x['message'] for x in logs
+                if BrowserLogLevel[x['level']] >= threshold
+                and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
+    assert messages == [], f'Messages with log level {threshold} or above in browser console'
+
+
+@pytest.mark.parametrize('level,expected',
+                         [
+                             ('ALL', True),
+                             ('SEVERE', True),
+                             ('OFF', False),
+                         ])
+@pytest.mark.internal
+def test_browser_log_level(level, expected):
+    """
+    Self test the logic to determine if browser log matches
+    """
+    assert (BrowserLogLevel.SEVERE >= BrowserLogLevel[level]) == expected