From a72d6eaefe6101f39f36d784e4f6d5b94323b698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Davorin=20Ru=C5=A1evljan?= Date: Thu, 14 Nov 2024 12:32:29 +0100 Subject: [PATCH] playwright coverage (#555) * playwright coverage WIP(1) * playwright coverage WIP(2) * playwright coverage WIP(3) * playwright coverage WIP(4) * playwright coverage WIP(5) * playwright coverage WIP(6) * playwright coverage WIP(7) * playwright coverage WIP(8) * playwright coverage WIP(9) * playwright coverage WIP(10) * playwright coverage WIP(11) * playwright coverage WIP(12) * playwright coverage WIP(12+1) * playwright coverage WIP(14) * playwright coverage WIP(15) * playwright coverage WIP(16) * playwright coverage WIP(17) * playwright coverage WIP(18) * playwright coverage WIP(19) * playwright coverage WIP(20) * playwright coverage WIP(21) * playwright coverage WIP(22) * playwright coverage WIP(23) * playwright coverage WIP(24) * added sleep after starting fastagency server in CI * wip * wip * wip * playwright coverage WIP2(1) * playwright coverage WIP2(2) * playwright coverage WIP2(3) * playwright coverage WIP2(3) * playwright coverage WIP2(5) * playwright coverage WIP2(6) * playwright coverage WIP2(7) * playwright coverage WIP2(8) * playwright coverage WIP2(10) --------- Co-authored-by: Davor Runje --- .github/workflows/pipeline.yaml | 18 ++- .github/workflows/test-playwright.yaml | 162 +++++++++++++++++++++++++ .github/workflows/test.yaml | 13 -- e2e/llm-sans/messages.spec.ts | 2 +- e2e/playwright.base.config.ts | 24 ++-- e2e/playwright.coverage.cfg | 26 ++++ fastagency/app.py | 13 ++ playwright.coverage.config.ts | 15 +++ playwright.gunicorn.config.ts | 2 +- playwright.llm-sans.config.ts | 8 +- 10 files changed, 256 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/test-playwright.yaml create mode 100644 e2e/playwright.coverage.cfg create mode 100644 playwright.coverage.config.ts diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index 1a7060265..19e14d0cc 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -57,6 +57,19 @@ jobs: use-llms: "" secrets: inherit # pragma: allowlist secret + test-playwright-without-llms: + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12"] + fail-fast: false + uses: ./.github/workflows/test-playwright.yaml + with: + python-version: ${{ matrix.python-version }} + environment: null + use-llms: "" + secrets: inherit # pragma: allowlist secret + + test-with-anthropic: uses: ./.github/workflows/test.yaml with: @@ -137,6 +150,7 @@ jobs: needs: - test-without-llms - test-with-llm + - test-playwright-without-llms - test-with-anthropic - test-with-azure_oai - test-with-openai @@ -165,8 +179,8 @@ jobs: - run: ls -la coverage - run: coverage combine coverage - - run: coverage report - - run: coverage html --show-contexts --title "FastAgency coverage for ${{ github.sha }}" + - run: coverage report -i + - run: coverage html -i --show-contexts --title "FastAgency coverage for ${{ github.sha }}" - name: Store coverage html uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-playwright.yaml b/.github/workflows/test-playwright.yaml new file mode 100644 index 000000000..2f7f88df9 --- /dev/null +++ b/.github/workflows/test-playwright.yaml @@ -0,0 +1,162 @@ +name: Internal test runner + +on: + workflow_call: + inputs: + environment: + description: 'Environment to run the tests in' + required: false + default: null + type: string + python-version: + description: 'Python version to run the tests in' + required: true + type: string + use-llms: + description: 'Use LLM in the tests' + required: false + type: string + default: "" + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: ${{ inputs.environment }} + services: + nats: + image: diementros/nats:js + ports: + - 4222:4222 + env: + NATS_URL: nats://localhost:4222 + steps: + - name: Set up environment variables + run: | + # check if an environment var or secret is defined and set env var to its value + + # vars + + if [ -n "${{ vars.AZURE_API_VERSION }}" ]; then + echo "AZURE_API_VERSION=${{ vars.AZURE_API_VERSION }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_API_ENDPOINT }}" ]; then + echo "AZURE_API_ENDPOINT=${{ vars.AZURE_API_ENDPOINT }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT35_MODEL }}" ]; then + echo "AZURE_GPT35_MODEL=${{ vars.AZURE_GPT35_MODEL }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT4_MODEL }}" ]; then + echo "AZURE_GPT4_MODEL=${{ vars.AZURE_GPT4_MODEL }}" >> $GITHUB_ENV + fi + if [ -n "${{ vars.AZURE_GPT4o_MODEL }}" ]; then + echo "AZURE_GPT4o_MODEL=${{ vars.AZURE_GPT4o_MODEL }}" >> $GITHUB_ENV + fi + + # secrets + + if [ -n "${{ secrets.AZURE_OPENAI_API_KEY }}" ]; then + echo "AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.TOGETHER_API_KEY }}" ]; then + echo "TOGETHER_API_KEY=${{ secrets.TOGETHER_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.OPENAI_API_KEY }}" ]; then + echo "OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.ANTHROPIC_API_KEY }}" ]; then + echo "ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.BING_API_KEY }}" ]; then + echo "BING_API_KEY=${{ secrets.BING_API_KEY }}" >> $GITHUB_ENV + fi + if [ -n "${{ secrets.POSTMAN_API_KEY }}" ]; then + echo "POSTMAN_API_KEY=${{ secrets.POSTMAN_API_KEY }}" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: "pip" + cache-dependency-path: pyproject.toml + - uses: actions/cache@v4 + id: cache + with: + path: ${{ env.pythonLocation }} + key: ${{ runner.os }}-python-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}-test-v03 + - name: Install Dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: pip install .[docs,testing] + - name: Install Pydantic v2 + run: pip install --pre "pydantic>=2,<3" + - run: mkdir coverage + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm ci + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Start fastagency + id: fastagency-start + run: | + # Start fastagency and grab its pid + nohup gunicorn --workers=1 e2e.llm-sans.main:app >nohup.txt 2>nohup-error.txt & + # Get the process ID (PID) + FAST_PID=$! + echo "Started fastagency with PID: $FAST_PID" + echo "FAST_PID=$FAST_PID" >> $GITHUB_OUTPUT + env: + COVERAGE_PROCESS_START: "e2e/playwright.coverage.cfg" + - run: echo "obtained FAST_PID" ${{ steps.fastagency-start.outputs.FAST_PID}} + - name: Run Playwright tests without LLMs + if: ${{ inputs.python-version != '3.9' }} + run: npx playwright test -c "playwright.coverage.config.ts" + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} + path: playwright-report/ + retention-days: 30 + + - name: Kill the program + run: | + PID="${{ steps.fastagency-start.outputs.FAST_PID}}" + # Send SIGTERM to the program (graceful shutdown) + echo "killing the fastagency:" $PID + ps -ef | grep $PID + kill -s SIGTERM $PID + + wait_count=0 + while ps -p "$PID" > /dev/null && ((wait_count < 5)); do + echo "Fastagency with PID $PID is still running..." + sleep 10 + wait_count=$((wait_count + 1)) + done + if ((wait_count == 5)); then + echo "Fastagency with PID $PID did not exit after 5 timeouts. Terminating..." + kill -9 $PID + else + echo "Fastagency with PID $PID has exited." + fi + + - run: ls -al .coverage + - run: cat nohup.txt + - run: cat nohup-error.txt + - name: Report + run: coverage report --no-skip-covered + - name: Move coverage file + run: mv .coverage coverage/.coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} + - name: Check coverage file + run: ls -al coverage + - name: Store coverage files + if: ${{ inputs.python-version != '3.9' }} + uses: actions/upload-artifact@v4 + with: + name: .coverage.playwright.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} + path: coverage/.coverage.playwright.* + if-no-files-found: error + overwrite: true + include-hidden-files: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 094f226ac..a67a693c1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -92,10 +92,6 @@ jobs: - uses: actions/setup-node@v4 with: node-version: lts/* - - name: Install dependencies - run: npm ci - - name: Install Playwright Browsers - run: npx playwright install --with-deps - name: Test without LLMs if: ${{ inputs.use-llms == ''}} run: bash scripts/test.sh -vv -m "not (anthropic or azure_oai or openai or togetherai or llm)" @@ -108,15 +104,6 @@ jobs: env: COVERAGE_FILE: coverage/.coverage.${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} CONTEXT: ${{ runner.os }}-py${{ inputs.python-version }}-${{ inputs.use-llms }} - - name: Run Playwright tests without LLMs - if: ${{ inputs.python-version != '3.9' }} - run: npx playwright test -c "playwright.llm-sans.config.ts" - - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: playwright-report - path: playwright-report/ - retention-days: 30 - name: Check coverage file run: ls -al coverage - name: Store coverage files diff --git a/e2e/llm-sans/messages.spec.ts b/e2e/llm-sans/messages.spec.ts index 220ad2005..29d9f30da 100644 --- a/e2e/llm-sans/messages.spec.ts +++ b/e2e/llm-sans/messages.spec.ts @@ -86,7 +86,7 @@ test('workflow started', async ({ page }) => { await page.goto('/'); const startWorkflow = await page.getByRole('button', { name: 'Workflow started' }) await startWorkflow.click() - const started = await page.getByText("Workflow started") + const started = await page.getByText("Workflow started: AutoGenWorkflows") await expect(started).toBeVisible() const name = await page.getByText('_workflow_started_') await expect(name).toBeVisible() diff --git a/e2e/playwright.base.config.ts b/e2e/playwright.base.config.ts index 203619dc5..1c194bd54 100644 --- a/e2e/playwright.base.config.ts +++ b/e2e/playwright.base.config.ts @@ -17,18 +17,21 @@ import { devices, defineConfig as originalDefineConfig, PlaywrightTestConfig } f * See https://playwright.dev/docs/test-configuration. */ +//const testEnv = { ...process.env, COVERAGE_PROCESS_START: 'playwright.coverage.cfg' } as { [key: string]: string } + + const baseConfig: PlaywrightTestConfig = { testDir: './e2e', /* Run tests in files in parallel */ - fullyParallel: true, + fullyParallel: false, /* Fail the build on CI if you accidentally left test.only in the source code. */ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, + workers: process.env.CI ? 1 : 1, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'list', + reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ @@ -76,12 +79,15 @@ const baseConfig: PlaywrightTestConfig = { // }, ], - /* Run your local dev server before starting the tests */ - webServer: { - command: 'fastagency run e2e/llm-sans/main.py', - url: 'http://127.0.0.1:32123', - reuseExistingServer: !process.env.CI, - }, + // /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'fastagency run e2e/llm-sans/main.py', + // url: 'http://127.0.0.1:32123', + // // env: testEnv, + // reuseExistingServer: true, + // //reuseExistingServer: !process.env.CI, + // //reuseExistingServer: false, + // }, } export function defineConfig(config: PlaywrightTestConfig): PlaywrightTestConfig { diff --git a/e2e/playwright.coverage.cfg b/e2e/playwright.coverage.cfg new file mode 100644 index 000000000..c1a0a04eb --- /dev/null +++ b/e2e/playwright.coverage.cfg @@ -0,0 +1,26 @@ +[coverage:run] +branch = True +sigterm = True +context = playwright + +[coverage:report] +omit = + /tmp/* + +exclude_also = + ; Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + ; Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + ; Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + + ; Don't complain about abstract methods, they aren't run: + @(abc\.)?abstractmethod + +ignore_errors = True diff --git a/fastagency/app.py b/fastagency/app.py index 3f00eddfd..2bba9ede7 100644 --- a/fastagency/app.py +++ b/fastagency/app.py @@ -1,5 +1,6 @@ __all__ = ["FastAgency"] +import os from collections.abc import Awaitable, Generator from contextlib import contextmanager from typing import Any, Callable, Optional, Union @@ -37,6 +38,18 @@ def __init__( title (Optional[str], optional): The title of the FastAgency. If None, the default string will be used. Defaults to None. description (Optional[str], optional): The description of the FastAgency. If None, the default string will be used. Defaults to None. """ + # check if we need to start coverage + logger.info("Checking if coverage is needed.") + coverage_process_start = os.environ.get("COVERAGE_PROCESS_START") + if coverage_process_start: + logger.info("Coverage process start detected") + logger.info(f"Coverage configuration file: {coverage_process_start}") + logger.info( + "To ensure coverage is written out, terminate this program with SIGTERM" + ) + import coverage + + coverage.process_startup() _self: Runnable = self self._title = title or "FastAgency application" self._description = description or "FastAgency application" diff --git a/playwright.coverage.config.ts b/playwright.coverage.config.ts new file mode 100644 index 000000000..6d38f98e4 --- /dev/null +++ b/playwright.coverage.config.ts @@ -0,0 +1,15 @@ +// playwright setting for tests that run with unicorn +import { defineConfig } from "./e2e/playwright.base.config.ts"; + +export default defineConfig( + { + testDir: './e2e/llm-sans', + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://127.0.0.1:8000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + } +) diff --git a/playwright.gunicorn.config.ts b/playwright.gunicorn.config.ts index 9081e8b03..3ba8c91a4 100644 --- a/playwright.gunicorn.config.ts +++ b/playwright.gunicorn.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "./e2e/playwright.base.config.ts"; export default defineConfig( { - testDir: './e2e/llm', + testDir: './e2e/llm-sans', use: { /* Base URL to use in actions like `await page.goto('/')`. */ baseURL: 'http://127.0.0.1:8000', diff --git a/playwright.llm-sans.config.ts b/playwright.llm-sans.config.ts index c513d28aa..60a9b2fba 100644 --- a/playwright.llm-sans.config.ts +++ b/playwright.llm-sans.config.ts @@ -3,5 +3,11 @@ import { defineConfig } from "./e2e/playwright.base.config.ts"; export default defineConfig( { testDir: './e2e/llm-sans', - } + webServer: { + command: 'fastagency run e2e/llm-sans/main.py', + url: 'http://127.0.0.1:32123', + // env: testEnv, + reuseExistingServer: true, + }, + }, )