From b9c9f4ec63823c12bfb46f52e5c2aa5885906abe Mon Sep 17 00:00:00 2001 From: Marco Date: Wed, 28 Jun 2023 23:08:40 +0200 Subject: [PATCH] test: Add Python script to test UIs --- scripts/release.sh | 27 +---- scripts/test-uis.py | 256 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 scripts/test-uis.py diff --git a/scripts/release.sh b/scripts/release.sh index 29c51d504d..7e05fc14f4 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -115,6 +115,8 @@ validate_default_project() { export hello_world_frontend_url="http://localhost:$webserver_port/?canisterId=$hello_world_frontend_canister_id" export candid_ui_url="http://localhost:$webserver_port/?canisterId=$candid_ui_id&id=$application_canister_id" + pip install playwright==1.35.0 + echo echo "==================================================" echo "dfx project directory: $(pwd)" @@ -122,30 +124,11 @@ validate_default_project() { echo "candid URL: $candid_ui_url" echo "==================================================" echo - echo "[1/4] Verify 'hello' functionality in a browser." - echo " - Open this URL in your web browser with empty cache or 'Private Browsing' mode" - echo " - Type a name and verify the response." - echo - echo " $hello_world_frontend_url" - echo - wait_for_response 'frontend UI passes' - echo - echo "[2/4] Verify there are no errors in the console by opening the Developer Tools." - echo - wait_for_response 'no errors on console' - echo - echo "[3/4] Verify the Candid UI." - echo - echo " - Open this URL in your web browser with empty cache or 'Private Browsing' mode" - echo " - Verify UI loads, then test the greet function by entering text and clicking *Call* or clicking *Lucky*" - echo - echo " $candid_ui_url" - echo - wait_for_response 'candid UI passes' + echo "Verify the Python script output." echo - echo "[4/4] Verify there are no errors in the console by opening the Developer Tools." + python3 scripts/test-uis.py --frontend_url "$hello_world_frontend_url" --candid_url "$candid_ui_url" --browsers chromium firefox webkit echo - wait_for_response 'no errors on console' + wait_for_response 'Python script logs are ok' echo dfx stop diff --git a/scripts/test-uis.py b/scripts/test-uis.py new file mode 100644 index 0000000000..220601f12b --- /dev/null +++ b/scripts/test-uis.py @@ -0,0 +1,256 @@ +''' +Automate frontend tests by using Playwright. +The script tests the following UIs: + +1. Frontend UI. +2. Candid UI. + +Examples: + +$ python3 test-uis.py --frontend_url '...' --browser chromium firefox webkit # Only test the frontend UI +$ python3 test-uis.py --candid_url '...' --browser chromium firefox webkit # Only test the Candid UI +$ python3 test-uis.py --frontend_url '...' --candid_url '...' --browser chromium firefox webkit # Test both UIs +''' +import argparse +import logging +import re +import sys + +from playwright.sync_api import sync_playwright + +_CHROMIUM_BROWSER = 'chromium' +_FIREFOX_BROWSER = 'firefox' +_WEBKIT_BROWSER = 'webkit' +_SUPPORTED_BROWSERS = { + _CHROMIUM_BROWSER, + _FIREFOX_BROWSER, + _WEBKIT_BROWSER, +} +_CANDID_UI_WARNINGS_TO_IGNORE = [ + ('Invalid asm.js: Unexpected token', '/index.js'), + ('Expected to find result for path [object Object], but instead found nothing.', '/index.js'), + (''' +Error: Server returned an error: + Code: 404 (Not Found) + Body: Custom section name not found. + + at j.readState (http://localhost:4943/index.js:2:11709) + at async http://localhost:4943/index.js:2:97683 + at async Promise.all (index 0) + at async Module.UA (http://localhost:4943/index.js:2:98732) + at async Object.getNames (http://localhost:4943/index.js:2:266156) + at async http://localhost:4943/index.js:2:275479'''.strip(), '/index.js') +] +_CANDID_UI_ERRORS_TO_IGNORE = [ + ('Failed to load resource: the server responded with a status of 404 (Not Found)', '/read_state'), +] + + +def _validate_browsers(browser): + if browser not in _SUPPORTED_BROWSERS: + logging.error(f'Browser {browser} not supported') + sys.exit(1) + + return browser + + +def _get_argument_parser(): + parser = argparse.ArgumentParser(description='Test the Frontend and Candid UIs') + + parser.add_argument('--frontend_url', help='Frontend UI url') + parser.add_argument('--candid_url', help='Candid UI url') + + parser.add_argument('--browsers', nargs='+', type=_validate_browsers, + help=f'Test against the specified browsers ({_SUPPORTED_BROWSERS})') + + return parser + + +def _validate_args(args): + has_err = False + + if not args.frontend_url and not args.candid_url: + logging.error('Either "--frontend_url" or "--candid_url" must be specified to start the tests') + has_err = True + + if not args.browsers: + logging.error('At least one browser must be specified') + logging.error(f'Possible browsers: {_SUPPORTED_BROWSERS}') + has_err = True + + if has_err: + sys.exit(1) + + +def _get_browser_obj(playwright, browser_name): + if browser_name == _CHROMIUM_BROWSER: + return playwright.chromium + if browser_name == _FIREFOX_BROWSER: + return playwright.firefox + if browser_name == _WEBKIT_BROWSER: + return playwright.webkit + + return None + + +def _check_console_logs(console_logs): + logging.info('Checking console logs') + + has_err = False + for log in console_logs: + if log.type not in {'warning', 'error'}: + continue + + # Skip all `Error with Permissions-Policy header: Unrecognized feature` warnings + perm_policy_warn = 'Error with Permissions-Policy header:' + if perm_policy_warn in log.text: + logging.warning(f'Skipping Permissions-Policy warning. log.text="{log.text}"') + continue + + url = log.location.get('url') + if not url: + raise RuntimeError(f'Cannot find "url" during log parsing (log.type={log.type}, log.text="{log.text}", log.location="{log.location}")') + + for actual_text, endpoint in (_CANDID_UI_ERRORS_TO_IGNORE if log.type == 'error' else _CANDID_UI_WARNINGS_TO_IGNORE): + if actual_text == log.text and endpoint in url: + logging.warning(f'Found {log.type}, but it was expected (log.type="{actual_text}", endpoint="{endpoint}")') + break + else: + logging.error(f'Found unexpected console log {log.type}. Text: "{log.text}"') + has_err = True + + if has_err: + raise RuntimeError('Console has unexpected warnings and/or errors. Check previous logs') + + logging.info('Console logs are ok') + + +def _click_button(page, button): + logging.info(f'Clicking button "{button}"') + page.get_by_role('button', name=button).click() + + +def _set_text(page, text, value): + logging.info(f'Setting text to "{value}"') + page.get_by_placeholder(text).fill(value) + + +def _test_frontend_ui_handler(browser, context, page): + # Set the name & Click the button + name = 'my name' + logging.info(f'Setting name "{name}"') + page.get_by_label('Enter your name:').fill(name) + _click_button(page, 'Click Me!') + + # Check if `#greeting` is populated correctly + greeting_id = '#greeting' + greeting_obj = page.query_selector(greeting_id) + if greeting_obj: + actual_value = greeting_obj.inner_text() + expected_value = f'Hello, {name}!' + if actual_value == expected_value: + logging.info(f'"{actual_value}" found in "{greeting_id}"') + else: + raise RuntimeError(f'Expected greeting message is "{expected_value}", but found "{actual_value}"') + else: + raise RuntimeError(f'Cannot find {greeting_id} selector') + + +def _test_candid_ui_handler(browser, context, page): + # Set the text & Click the "Query" button + text = 'hello, world' + _set_text(page, 'text', text) + _click_button(page, 'Query') + + # Reset the text & Click the "Random" button + _set_text(page, 'text', '') + _click_button(page, 'Random') + # ~ + + # Check if `#output-list` is populated correctly + output_list_id = '#output-list' + output_list_obj = page.query_selector(output_list_id) + if output_list_obj: + output_list_lines = output_list_obj.inner_text().split('\n') + actual_num_lines, expected_num_lines = len(output_list_lines), 4 + if actual_num_lines != expected_num_lines: + raise RuntimeError(f'Expected {expected_num_lines} lines of text but found {actual_num_lines}') + + # Extract random text from third line + random_text = re.search(r'"([^"]*)"', output_list_lines[2]) + if not random_text: + raise RuntimeError(f'Cannot extract the random text from the third line: {output_list_lines[2]}') + random_text = random_text.group(1) + + for i, text_str in enumerate([text, random_text]): + l1, l2 = (i * 2), (i * 2 + 1) + + # First output line + actual_line, expected_line = output_list_lines[l1], f'› greet("{text_str}")' + if actual_line != expected_line: + raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l1})') + logging.info(f'"{actual_line}" found in {output_list_id} at position {l1}') + + # Second output line + actual_line, expected_line = output_list_lines[l2], f'("Hello, {text_str}!")' + if actual_line != expected_line: + raise RuntimeError(f'Expected {expected_line} line, but found {actual_line} (line {l2})') + logging.info(f'"{actual_line}" found in {output_list_id} at position {l2}') + + logging.info(f'{output_list_id} lines are defined correctly') + else: + raise RuntimeError(f'Cannot find {output_list_id} selector') + + +def _test_ui(url, ui_name, handler, browsers): + logging.info(f'Testing "{ui_name}" at "{url}"') + + has_err = False + with sync_playwright() as playwright: + for browser_name in browsers: + logging.info(f'Checking "{browser_name}" browser') + browser = _get_browser_obj(playwright, browser_name) + if not browser: + raise RuntimeError(f'Cannot determine browser object for browser {browser_name}') + + try: + browser = playwright.chromium.launch(headless=True) + context = browser.new_context() + page = context.new_page() + + # Attach a listener to the page's console events + console_logs = [] + page.on('console', lambda msg: console_logs.append(msg)) + + page.goto(url) + + handler(browser, context, page) + _check_console_logs(console_logs) + except Exception as e: + logging.error(f'Error: {str(e)}') + has_err = True + finally: + if context: + context.close() + if browser: + browser.close() + + if has_err: + sys.exit(1) + + +def _main(): + args = _get_argument_parser().parse_args() + _validate_args(args) + + if args.frontend_url: + _test_ui(args.frontend_url, 'Frontend UI', _test_frontend_ui_handler, args.browsers) + if args.candid_url: + _test_ui(args.candid_url, 'Candid UI', _test_candid_ui_handler, args.browsers) + + logging.info('DONE!') + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + _main()