Make browser log level threshold configurable

Previously it was tuned to only trigger on SEVERE messages, but the
desire is to make it stricter and also fail tests when there are
warnings. By making it a variable, it can be changed without changing
the code.

tests/conftest.py

@@ -8,6 +8,10 @@
 User = namedtuple('User', ['username', 'password', 'name'])
 
 
+def pytest_configure(config):
+    config.addinivalue_line('markers', 'internal: mark test as a self test')
+
+
 def pytest_generate_tests(metafunc):
     variables = metafunc.config._variables  # pylint: disable=protected-access
 

tests/test_pages.py

@@ -1,6 +1,7 @@
 # pylint: disable=redefined-outer-name
 from __future__ import print_function
 
+from enum import IntFlag
 from typing import Union, KeysView, List
 from urllib.parse import ParseResult, urlparse, parse_qs, urlencode
 
@@ -12,6 +13,10 @@
 from selenium.common.exceptions import TimeoutException
 
 
+# https://github.com/SeleniumHQ/selenium/wiki/Logging
+BrowserLogLevel = IntFlag('BrowserLogLevel', ['ALL', 'DEBUG', 'INFO', 'WARNING', 'SEVERE', 'OFF'])
+
+
 EXCLUDE_ERRORS = (
     # Fixed in Foreman 3.7 - https://projects.theforeman.org/issues/36093
     'Scrollbar test exception: TypeError:',
@@ -52,7 +57,7 @@ def filtered_url_query(url: str, allowed_query_params: Union[List, KeysView]) ->
 
 @pytest.mark.nondestructive
 @pytest.mark.selenium
-def test_menu_item(selenium, user, url):
+def test_menu_item(selenium, user, url, variables):
     selenium.get(url)
     assert selenium.current_url.endswith('/users/login'), 'Redirect to login page'
     login_field = selenium.find_element(By.NAME, 'login[login]')
@@ -82,5 +87,22 @@ def test_menu_item(selenium, user, url):
         print("https://github.com/mozilla/geckodriver/issues/284")
 
     logs = selenium.get_log('browser')
-    severe_messages = [x['message'] for x in logs if x.get('level') == 'SEVERE' and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
-    assert severe_messages == [], 'Error messages with log level SEVERE in browser console'
+    threshold = BrowserLogLevel[variables.get('browser_log_threshold', 'SEVERE')]
+    messages = [x['message'] for x in logs
+                if BrowserLogLevel[x['level']] >= threshold
+                and not any(excl in x['message'] for excl in EXCLUDE_ERRORS)]
+    assert messages == [], f'Messages with log level {threshold} or above in browser console'
+
+
+@pytest.mark.parametrize('level,expected',
+                         [
+                             ('ALL', True),
+                             ('SEVERE', True),
+                             ('OFF', False),
+                         ])
+@pytest.mark.internal
+def test_browser_log_level(level, expected):
+    """
+    Self test the logic to determine if browser log matches
+    """
+    assert (BrowserLogLevel.SEVERE >= BrowserLogLevel[level]) == expected