Add support for custom flow decorators to prefect deploy
#15841
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Unit tests | |
env: | |
# enable colored output | |
# https://github.com/pytest-dev/pytest/issues/7443 | |
PY_COLORS: 1 | |
on: | |
pull_request: | |
paths: | |
- .github/workflows/python-tests.yaml | |
- "src/prefect/**/*.py" | |
- "tests/**/*.py" | |
- requirements.txt | |
- requirements-client.txt | |
- requirements-dev.txt | |
- setup.cfg | |
- Dockerfile | |
push: | |
branches: | |
- 2.x | |
paths: | |
- .github/workflows/python-tests.yaml | |
- "src/prefect/**/*.py" | |
- "tests/**/*.py" | |
- requirements.txt | |
- requirements-client.txt | |
- requirements-dev.txt | |
- setup.cfg | |
- Dockerfile | |
permissions: | |
contents: read | |
# Limit concurrency by workflow/branch combination. | |
# | |
# For pull request builds, pushing additional changes to the | |
# branch will cancel prior in-progress and pending builds. | |
# | |
# For builds triggered on a branch push, additional changes | |
# will wait for prior builds to complete before starting. | |
# | |
# https://docs.github.com/en/actions/using-jobs/using-concurrency | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
jobs: | |
run-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: python:${{ matrix.python-version }}, ${{ matrix.database }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
- "sqlite" | |
python-version: | |
- "3.8" | |
- "3.9" | |
- "3.10" | |
- "3.11" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'pydantic>=2.4,<3' | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
# Parallelize tests by scope to reduce expensive service fixture duplication | |
# Do not allow the test suite to build images, as we want the prebuilt images to be tested | |
# Do not run Kubernetes service tests, we do not have a cluster available | |
# maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--dist loadscope | |
--disable-docker-image-builds | |
--exclude-service kubernetes | |
--exclude-service docker | |
--durations 26 | |
--no-cov | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: > | |
docker container inspect postgres | |
&& docker container logs postgres | |
|| echo "Ignoring bad exit code" | |
run-pydantic-v1-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: pydantic v1, python:${{ matrix.python-version }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
python-version: | |
- "3.8" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'pydantic<2' | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
# Parallelize tests by scope to reduce expensive service fixture duplication | |
# Do not allow the test suite to build images, as we want the prebuilt images to be tested | |
# Do not run Kubernetes service tests, we do not have a cluster available | |
# maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--dist loadscope | |
--disable-docker-image-builds | |
--exclude-service kubernetes | |
--exclude-service docker | |
--durations 26 | |
--no-cov | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="pydantic-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.database }}-${{ matrix.pytest-options }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: > | |
docker container inspect postgres | |
&& docker container logs postgres | |
|| echo "Ignoring bad exit code" | |
run-pydantic-v2-internals-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: pydantic v2 internals, python:${{ matrix.python-version }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
python-version: | |
- "3.8" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Get image tag | |
id: get_image_tag | |
run: | | |
SHORT_SHA=$(git rev-parse --short=7 HEAD) | |
tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}" | |
echo "image_tag=${tmp}" >> $GITHUB_OUTPUT | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'pydantic>=2.4,<3' | |
- name: Set PREFECT_EXPERIMENTAL_ENABLE_PYDANTIC_V2_INTERNALS env var | |
run: echo "PREFECT_EXPERIMENTAL_ENABLE_PYDANTIC_V2_INTERNALS=1" >> $GITHUB_ENV | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
# Parallelize tests by scope to reduce expensive service fixture duplication | |
# Do not allow the test suite to build images, as we want the prebuilt images to be tested | |
# Do not run Kubernetes service tests, we do not have a cluster available | |
# maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--dist loadscope | |
--disable-docker-image-builds | |
--exclude-service kubernetes | |
--exclude-service docker | |
--durations 26 | |
--no-cov | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="pydantic-v2-internals-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.database }}-${{ matrix.pytest-options }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: > | |
docker container inspect postgres | |
&& docker container logs postgres | |
|| echo "Ignoring bad exit code" | |
run-sqlalchemy-v1-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: sqlalchemy v1, ${{ matrix.database }} python:${{ matrix.python-version }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
- "sqlite" | |
python-version: | |
- "3.8" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'sqlalchemy[asyncio]<2' | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
# Parallelize tests by scope to reduce expensive service fixture duplication | |
# Do not allow the test suite to build images, as we want the prebuilt images to be tested | |
# Do not run Kubernetes service tests, we do not have a cluster available | |
# maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--dist loadscope | |
--disable-docker-image-builds | |
--exclude-service kubernetes | |
--exclude-service docker | |
--durations 26 | |
--no-cov | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="sqlalchemy-v1-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.database }}-${{ matrix.pytest-options }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: > | |
docker container inspect postgres | |
&& docker container logs postgres | |
|| echo "Ignoring bad exit code" | |
notify-tests-failing-on-main: | |
needs: run-tests | |
if: github.ref == 'refs/heads/main' && failure() | |
runs-on: ubuntu-latest | |
env: | |
FAILURE_THRESHOLD: 1 | |
steps: | |
- name: Download all failure flags | |
uses: actions/download-artifact@v4 | |
with: | |
path: failure-flags/ | |
- name: Check for failure flags | |
id: check_failure | |
run: | | |
failure_count=$(ls -1q failure-flags/*/*.txt | wc -l) | |
if [ $failure_count -gt $FAILURE_THRESHOLD ]; then | |
too_many_tests_failed="true" | |
else | |
too_many_tests_failed="false" | |
fi | |
echo "failure_count=$failure_count" >> $GITHUB_OUTPUT | |
echo "too_many_tests_failed=$too_many_tests_failed" >> $GITHUB_OUTPUT | |
- name: Send Slack Notification | |
if: ${{ steps.check_failure.outputs.too_many_tests_failed == 'true' }} | |
uses: 8398a7/action-slack@v3 | |
with: | |
author_name: Prefect OSS Tests Failing on Main | |
channel: CBH18KG8G # This is #engineering | |
fields: message,commit,author,workflowRun | |
status: failure | |
text: ":warning: Unit tests are failing in Prefect's main branch. Commit author: please either fix or remove the failing tests. If you remove the failing tests create a GitHub issue with the details." | |
env: | |
SLACK_WEBHOOK_URL: ${{ secrets.ENGINEERING_REVIEW_SLACK_WEBHOOK_URL }} | |
run-tests-for-datadog: | |
name: DataDog CI Visibility | |
if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
runs-on: | |
group: oss-larger-runners | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
python-version: | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
with: | |
driver-opts: image=moby/buildkit:v0.12.5 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Get image tag | |
id: get_image_tag | |
run: | | |
SHORT_SHA=$(git rev-parse --short=7 HEAD) | |
tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}" | |
echo "image_tag=${tmp}" >> $GITHUB_OUTPUT | |
- name: Build test image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
build-args: | | |
PYTHON_VERSION=${{ matrix.python-version }} | |
PREFECT_EXTRAS=[dev] | |
tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} | |
outputs: type=docker,dest=/tmp/image.tar | |
- name: Test Docker image | |
run: | | |
docker load --input /tmp/image.tar | |
docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'pydantic>=2.4,<3' | |
- name: Start database container | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--ddtrace | |
--dist loadscope | |
--disable-docker-image-builds | |
--exclude-service kubernetes | |
--durations 26 | |
--cov | |
--cov-config setup.cfg | |
env: | |
DD_CIVISIBILITY_AGENTLESS_ENABLED: true | |
DD_API_KEY: ${{ secrets.DD_API_KEY_CI_VISIBILITY }} | |
DD_SITE: datadoghq.com | |
DD_ENV: ci | |
DD_SERVICE: prefect | |
run-docker-tests: | |
runs-on: | |
group: oss-larger-runners | |
name: docker, python:${{ matrix.python-version }} | |
strategy: | |
matrix: | |
database: | |
- "postgres:14" | |
python-version: | |
- "3.8" | |
- "3.9" | |
- "3.10" | |
- "3.11" | |
- "3.12" | |
fail-fast: true | |
timeout-minutes: 45 | |
steps: | |
- name: Display current test matrix | |
run: echo '${{ toJSON(matrix) }}' | |
- uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
fetch-depth: 0 | |
- name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
with: | |
driver-opts: image=moby/buildkit:v0.12.5 | |
- name: Set up Python ${{ matrix.python-version }} | |
uses: actions/setup-python@v5 | |
id: setup_python | |
with: | |
python-version: ${{ matrix.python-version }} | |
- name: UV Cache | |
# Manually cache the uv cache directory | |
# until setup-python supports it: | |
# https://github.com/actions/setup-python/issues/822 | |
uses: actions/cache@v4 | |
id: cache-uv | |
with: | |
path: ~/.cache/uv | |
key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles('requirements-client.txt', 'requirements.txt', 'requirements-dev.txt') }} | |
- name: Get image tag | |
id: get_image_tag | |
run: | | |
SHORT_SHA=$(git rev-parse --short=7 HEAD) | |
tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}" | |
echo "image_tag=${tmp}" >> $GITHUB_OUTPUT | |
- name: Build test image | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
build-args: | | |
PYTHON_VERSION=${{ matrix.python-version }} | |
PREFECT_EXTRAS=[dev] | |
tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} | |
outputs: type=docker,dest=/tmp/image.tar | |
- name: Test Docker image | |
run: | | |
docker load --input /tmp/image.tar | |
docker run --rm prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }} prefect version | |
- name: Install packages | |
run: | | |
python -m pip install -U uv | |
uv pip install --upgrade --system -e .[dev] 'pydantic>=2.4,<3' | |
- name: Start database container | |
if: ${{ startsWith(matrix.database, 'postgres') }} | |
run: > | |
docker run | |
--name "postgres" | |
--detach | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--publish 5432:5432 | |
--tmpfs /var/lib/postgresql/data | |
--env POSTGRES_USER="prefect" | |
--env POSTGRES_PASSWORD="prefect" | |
--env POSTGRES_DB="prefect" | |
--env LANG="C.UTF-8" | |
--env LANGUAGE="C.UTF-8" | |
--env LC_ALL="C.UTF-8" | |
--env LC_COLLATE="C.UTF-8" | |
--env LC_CTYPE="C.UTF-8" | |
${{ matrix.database }} | |
-c max_connections=250 | |
./scripts/wait-for-healthy-container.sh postgres 30 | |
echo "PREFECT_API_DATABASE_CONNECTION_URL=postgresql+asyncpg://prefect:prefect@localhost/prefect" >> $GITHUB_ENV | |
# Parallelize tests by scope to reduce expensive service fixture duplication | |
# Do not allow the test suite to build images, as we want the prebuilt images to be tested | |
# Do not run Kubernetes service tests, we do not have a cluster available | |
# maxprocesses 6 is based on empirical testing; higher than 6 sees diminishing returns | |
- name: Run tests | |
run: > | |
pytest tests | |
--numprocesses auto | |
--maxprocesses 6 | |
--dist loadscope | |
--disable-docker-image-builds | |
--only-service docker | |
--durations 26 | |
--no-cov | |
- name: Create and Upload failure flag | |
if: ${{ failure() }} | |
id: create_failure_flag | |
run: | | |
sanitized_name="${{ matrix.python-version }}-${{ matrix.database }}" | |
sanitized_name="${sanitized_name//:/-}" | |
echo "Failure in $sanitized_name" > "${sanitized_name}-failure.txt" | |
echo "artifact_name=${sanitized_name}-failure" >> $GITHUB_OUTPUT | |
- name: Upload failure flag | |
if: ${{ failure() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: ${{ steps.create_failure_flag.outputs.artifact_name }} | |
path: "${{ steps.create_failure_flag.outputs.artifact_name }}.txt" | |
- name: Check database container | |
# Only applicable for Postgres, but we want this to run even when tests fail | |
if: always() | |
run: > | |
docker container inspect postgres | |
&& docker container logs postgres | |
|| echo "Ignoring bad exit code" |