Temporarily disable passing worker labels to flow run (#16339) #18441
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Unit tests | |
env: | |
# enable colored output | |
# https://github.com/pytest-dev/pytest/issues/7443 | |
PY_COLORS: 1 | |
on: | |
pull_request: | |
paths: | |
- .github/workflows/python-tests.yaml | |
- "src/prefect/**/*.py" | |
- "tests/**/*.py" | |
- requirements.txt | |
- requirements-client.txt | |
- requirements-dev.txt | |
- setup.cfg | |
- Dockerfile | |
push: | |
branches: | |
- main | |
paths: | |
- .github/workflows/python-tests.yaml | |
- "src/prefect/**/*.py" | |
- "tests/**/*.py" | |
- requirements.txt | |
- requirements-client.txt | |
- requirements-dev.txt | |
- setup.cfg | |
- Dockerfile | |
permissions: | |
contents: read | |
actions: write | |
# Limit concurrency by workflow/branch combination. | |
# | |
# For pull request builds, pushing additional changes to the | |
# branch will cancel prior in-progress and pending builds. | |
# | |
# For builds triggered on a branch push, additional changes | |
# will wait for prior builds to complete before starting. | |
# | |
# https://docs.github.com/en/actions/using-jobs/using-concurrency | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
jobs: | |
run-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: ${{ matrix.test-type.name }} - python:${{ matrix.python-version }}, ${{ matrix.database }} | |
strategy: | |
matrix: | |
test-type: | |
- name: Server Tests | |
modules: tests/server/ tests/events/server | |
- name: Client Tests | |
modules: tests/ --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers | |
- name: Runner and Worker Tests | |
modules: tests/test_task_runners.py tests/runner tests/workers | |
database: | |
- "postgres:14" | |
- "sqlite" | |
python-version: | |
- "3.9" | |
- "3.10" | |
- "3.11" | |
- "3.12" | |
exclude: | |
- database: "sqlite" | |
test-type: | |
name: Client Tests | |
modules: tests/ --ignore=tests/server/ --ignore=tests/events/server --ignore=tests/test_task_runners.py --ignore=tests/runner --ignore=tests/workers | |
- database: "sqlite" | |
test-type: | |
name: Runner and Worker Tests | |
modules: tests/test_task_runners.py tests/runner tests/workers | |
fail-fast: true | |
timeout-minutes: 15 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
- name: Start redis | |
run: > | |
docker run | |
--name "redis" | |
--detach | |
--publish 6379:6379 | |
redis:latest | |
- name: Set coverage file and artifact name | |
id: set_coverage_and_artifact_name | |
run: | | |
sanitized_test_type="${{ matrix.test-type.name }}" | |
sanitized_test_type="${sanitized_test_type// /_}" | |
sanitized_database="${{ matrix.database }}" | |
sanitized_database="${sanitized_database//:/\-}" | |
sanitized_python_version="${{ matrix.python-version }}" | |
export COVERAGE_FILE=".coverage.${sanitized_test_type}.${sanitized_python_version}.${sanitized_database}" | |
echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV | |
echo "artifact_name=coverage-data-${sanitized_test_type}-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT | |
- name: Set coverage core | |
if: ${{ matrix.python-version == '3.12' }} | |
run: | | |
echo "COVERAGE_CORE=sysmon" >> $GITHUB_ENV | |
- name: Run tests | |
run: | | |
echo "Using COVERAGE_FILE=$COVERAGE_FILE" | |
pytest ${{ matrix.test-type.modules }} \ | |
--numprocesses auto \ | |
--maxprocesses 6 \ | |
--dist worksteal \ | |
--disable-docker-image-builds \ | |
--exclude-service kubernetes \ | |
--exclude-service docker \ | |
--durations 26 \ | |
--cov=prefect \ | |
--cov-config=setup.cfg \ | |
--cov-report='' | |
- name: Upload coverage data | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }} | |
path: ${{ env.COVERAGE_FILE }} | |
include-hidden-files: true | |
retention-days: 1 | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: | | |
docker container inspect postgres \ | |
&& docker container logs postgres \ | |
|| echo "Ignoring bad exit code" | |
run-docker-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: docker, python:${{ matrix.python-version }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
python-version: | |
- "3.9" | |
- "3.10" | |
- "3.11" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
with: | |
driver-opts: image=moby/buildkit:v0.12.5 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Get image tag | |
id: get_image_tag | |
run: | | |
SHORT_SHA=$(git rev-parse --short=7 HEAD) | |
tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}" | |
echo "image_tag=${tmp}" >> $GITHUB_OUTPUT | |
- name: Get node version | |
id: get_node_version | |
run: | | |
echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_OUTPUT | |
- name: Login to DockerHub | |
uses: docker/login-action@v3 | |
if: github.event.pull_request.head.repo.full_name == github.repository | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_TOKEN }} | |
- name: Build test image | |
uses: docker/build-push-action@v6 | |
env: | |
DOCKER_BUILD_SUMMARY: false | |
DOCKER_RECORD_UPLOAD: false | |
with: | |
context: . | |
build-args: | | |
PYTHON_VERSION=${{ matrix.python-version }} | |
PREFECT_EXTRAS=[dev] | |
NODE_VERSION=${{ steps.get_node_version.outputs.NODE_VERSION }} | |
tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} | |
outputs: type=docker,dest=/tmp/image.tar | |
- name: Test Docker image | |
run: | | |
docker load --input /tmp/image.tar | |
docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_SERVER_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
- name: Start docker registry | |
run: > | |
docker run | |
--name "prefect-test-registry" | |
--detach | |
--publish 5555:5000 | |
registry:2 | |
- name: Start redis | |
run: > | |
docker run | |
--name "redis" | |
--detach | |
--publish 6379:6379 | |
redis:latest | |
- name: Set coverage file and artifact name | |
id: set_coverage_and_artifact_name | |
run: | | |
sanitized_database="${{ matrix.database }}" | |
sanitized_database="${sanitized_database//:/\-}" | |
sanitized_python_version="${{ matrix.python-version }}" | |
export COVERAGE_FILE=".coverage.${sanitized_python_version}.${sanitized_database}" | |
echo "COVERAGE_FILE=${COVERAGE_FILE}" >> $GITHUB_ENV | |
echo "artifact_name=coverage-data-docker-${{ matrix.python-version }}-${sanitized_database}" >> $GITHUB_OUTPUT | |
- name: Set coverage core | |
if: ${{ matrix.python-version == '3.12' }} | |
run: | | |
echo "COVERAGE_CORE=sysmon" >> $GITHUB_ENV | |
- name: Run tests | |
run: | | |
echo "Using COVERAGE_FILE=$COVERAGE_FILE" | |
pytest tests \ | |
--numprocesses auto \ | |
--maxprocesses 6 \ | |
--dist worksteal \ | |
--disable-docker-image-builds \ | |
--only-service docker \ | |
--durations 26 \ | |
--cov=prefect \ | |
--cov-config=setup.cfg \ | |
--cov-report='' | |
- name: Upload coverage data | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.set_coverage_and_artifact_name.outputs.artifact_name }} | |
path: ${{ env.COVERAGE_FILE }} | |
include-hidden-files: true | |
retention-days: 1 | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: | | |
docker container inspect postgres \ | |
&& docker container logs postgres \ | |
|| echo "Ignoring bad exit code" | |
combine-coverage: | |
runs-on: ubuntu-latest | |
needs: | |
- run-tests | |
- run-docker-tests | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
- name: Set up Python 3.12 | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: "3.12" | |
- name: Download coverage data artifacts | |
uses: actions/download-artifact@v4 | |
with: | |
pattern: coverage-data-* | |
merge-multiple: true | |
- name: Install coverage | |
run: pip install coverage | |
- name: Combine coverage data | |
run: coverage combine | |
- name: Generate HTML coverage report | |
run: coverage html | |
- name: Upload combined coverage report | |
uses: actions/upload-artifact@v4 | |
with: | |
name: combined-coverage-report | |
path: htmlcov/ | |
- name: Publish coverage markdown report | |
run: | | |
echo "## Coverage Report" >> $GITHUB_STEP_SUMMARY | |
echo "[Detailed Report](${{ steps.upload_combined_coverage_report.outputs.artifact_url }})" >> $GITHUB_STEP_SUMMARY | |
coverage report --format=markdown >> $GITHUB_STEP_SUMMARY |