Skip to content

Commit

Permalink
Merge branch 'main' into dependabot/pip/typer-gte-0.12.0-and-neq-0.12…
Browse files Browse the repository at this point in the history
….2-and-lt-0.16.0
  • Loading branch information
zzstoatzz authored Dec 13, 2024
2 parents 2af925d + 40590a5 commit 39bf7bf
Show file tree
Hide file tree
Showing 573 changed files with 31,752 additions and 22,191 deletions.
9 changes: 8 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ env/
venv/

# Documentation artifacts
docs/api-ref/schema.json
docs/
site/

# UI artifacts
src/prefect/server/ui/*
ui/node_modules
ui-v2/

# Databases
*.db
Expand All @@ -49,3 +50,9 @@ dask-worker-space/
# Editors
.idea/
.vscode/

# Other
tests/
compat-tests/
benches/
build/
2 changes: 1 addition & 1 deletion .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ updates:
directory: '/ui/'
schedule:
interval: "weekly"
labels: ["ui", "development"]
labels: ["ui", "ui-dependency"]

- package-ecosystem: "github-actions"
directory: "/"
Expand Down
4 changes: 4 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@ upstream dependency:

2.x:
- base-branch: '2.x'

ui-replatform:
- changed-files:
- any-glob-to-any-file: ui-v2/**
6 changes: 6 additions & 0 deletions .github/release.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
# .github/release.yml

changelog:
exclude:
labels:
- ui-dependency # these dont affect user environments
- ui-replatform # lots of ongoing work, but not user facing... yet
categories:
- title: Breaking Changes ⚠️
labels:
Expand All @@ -26,6 +30,8 @@ changelog:
- title: Development & Tidiness 🧹
labels:
- development
- title: Documentation 📓
labels:
- docs
- title: Uncategorized
labels:
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/docker-images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,19 @@ jobs:
flavor: |
latest=false
- name: Get node version
id: get_node_version
run: |
echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_OUTPUT
- name: Build and push image
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
build-args: |
PYTHON_VERSION=${{ matrix.python-version }}
NODE_VERSION=${{ steps.get_node_version.outputs.NODE_VERSION }}
${{ ( endsWith(matrix.flavor, 'conda') && 'BASE_IMAGE=prefect-conda' ) || '' }}
${{ ( endsWith(matrix.flavor, 'kubernetes') && 'PREFECT_EXTRAS=[kubernetes]' ) || '' }}
tags: ${{ join(steps.metadata-dev.outputs.tags) }},${{ join(steps.metadata-prod.outputs.tags) }}
Expand Down
74 changes: 74 additions & 0 deletions .github/workflows/proxy-test.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
# This is a simple test to ensure we can make a websocket connection through a proxy server. It sets up a
# simple server and a squid proxy server. The proxy server is inaccessible from the host machine, only the proxy
# so we can confirm the proxy is actually working.

name: Proxy Test
on:
pull_request:
paths:
- .github/workflows/proxy-test.yaml
- scripts/proxy-test/*
- "src/prefect/events/clients.py"
- requirements.txt
- requirements-client.txt
- requirements-dev.txt
push:
branches:
- main
paths:
- .github/workflows/proxy-test.yaml
- scripts/proxy-test/*
- "src/prefect/events/clients.py"
- requirements.txt
- requirements-client.txt
- requirements-dev.txt

jobs:
proxy-test:
name: Proxy Test
timeout-minutes: 10
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
fetch-depth: 0

- name: Set up Python 3.10
uses: actions/setup-python@v5
id: setup_python
with:
python-version: "3.10"

- name: Create Docker networks
run: |
docker network create internal_net --internal
docker network create external_net
- name: Start API server container
working-directory: scripts/proxy-test
run: |
docker build -t api-server .
docker run -d --network internal_net --name server api-server
- name: Start Squid Proxy container
run: |
docker run -d \
--network internal_net \
--network external_net \
-p 3128:3128 \
-v $(pwd)/scripts/proxy-test/squid.conf:/etc/squid/squid.conf \
--name proxy \
ubuntu/squid
- name: Install Dependencies
run: |
python -m pip install -U uv
uv pip install --upgrade --system .
- name: Run Proxy Tests
env:
HTTP_PROXY: http://localhost:3128
HTTPS_PROXY: http://localhost:3128
run: python scripts/proxy-test/client.py
9 changes: 8 additions & 1 deletion .github/workflows/python-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ on:
- requirements-dev.txt
- setup.cfg
- Dockerfile
- scripts/entrypoint.sh
push:
branches:
- main
Expand All @@ -28,7 +29,7 @@ on:
- requirements-dev.txt
- setup.cfg
- Dockerfile

- scripts/entrypoint.sh
permissions:
contents: read
actions: write
Expand Down Expand Up @@ -266,6 +267,11 @@ jobs:
tmp="sha-$SHORT_SHA-python${{ matrix.python-version }}"
echo "image_tag=${tmp}" >> $GITHUB_OUTPUT
- name: Get node version
id: get_node_version
run: |
echo "NODE_VERSION=$(cat .nvmrc)" >> $GITHUB_OUTPUT
- name: Login to DockerHub
uses: docker/login-action@v3
if: github.event.pull_request.head.repo.full_name == github.repository
Expand All @@ -283,6 +289,7 @@ jobs:
build-args: |
PYTHON_VERSION=${{ matrix.python-version }}
PREFECT_EXTRAS=[dev]
NODE_VERSION=${{ steps.get_node_version.outputs.NODE_VERSION }}
tags: prefecthq/prefect-dev:${{ steps.get_image_tag.outputs.image_tag }}
outputs: type=docker,dest=/tmp/image.tar

Expand Down
59 changes: 59 additions & 0 deletions .github/workflows/static-analysis.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,62 @@ jobs:
- name: Run pre-commit
run: |
pre-commit run --show-diff-on-failure --color=always --all-files
type-completeness-check:
name: Type completeness check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
persist-credentials: false
fetch-depth: 0

- name: Set up uv
uses: astral-sh/setup-uv@v4
with:
python-version: "3.12"

- name: Calculate type completeness score
id: calculate_current_score
run: |
# `pyright` will exit with a non-zero status code if it finds any issues,
# so we need to explicitly ignore the exit code with `|| true`.
uv tool run --with-editable . pyright --verifytypes prefect --ignoreexternal --outputjson > prefect-analysis.json || true
SCORE=$(jq -r '.typeCompleteness.completenessScore' prefect-analysis.json)
echo "current_score=$SCORE" >> $GITHUB_OUTPUT
- name: Checkout base branch
run: |
git checkout ${{ github.base_ref }}
- name: Calculate base branch score
id: calculate_base_score
run: |
uv tool run --with-editable . pyright --verifytypes prefect --ignoreexternal --outputjson > prefect-analysis-base.json || true
BASE_SCORE=$(jq -r '.typeCompleteness.completenessScore' prefect-analysis-base.json)
echo "base_score=$BASE_SCORE" >> $GITHUB_OUTPUT
- name: Compare scores
run: |
CURRENT_SCORE=$(echo ${{ steps.calculate_current_score.outputs.current_score }})
BASE_SCORE=$(echo ${{ steps.calculate_base_score.outputs.base_score }})
if (( $(echo "$BASE_SCORE > $CURRENT_SCORE" | bc -l) )); then
echo "::notice title=Type Completeness Check::We noticed a decrease in type coverage with these changes. Check workflow summary for more details."
echo "### ℹ️ Type Completeness Check" >> $GITHUB_STEP_SUMMARY
echo "We noticed a decrease in type coverage with these changes. To maintain our codebase quality, we aim to keep or improve type coverage with each change." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Need help? Ping @desertaxle or @zzstoatzz for assistance!" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Here's what changed:" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
uv run scripts/pyright_diff.py prefect-analysis-base.json prefect-analysis.json >> $GITHUB_STEP_SUMMARY
SCORE_DIFF=$(echo "$BASE_SCORE - $CURRENT_SCORE" | bc -l)
if (( $(echo "$SCORE_DIFF > 0.001" | bc -l) )); then
exit 1
fi
elif (( $(echo "$BASE_SCORE < $CURRENT_SCORE" | bc -l) )); then
echo "🎉 Great work! The type coverage has improved with these changes" >> $GITHUB_STEP_SUMMARY
else
echo "✅ Type coverage maintained" >> $GITHUB_STEP_SUMMARY
fi
112 changes: 112 additions & 0 deletions .github/workflows/time-docker-build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
name: Docker Build Time Benchmark

on:
push:
branches:
- main
paths:
- "Dockerfile"
- ".dockerignore"
pull_request:
paths:
- "Dockerfile"
- ".dockerignore"

jobs:
benchmark:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

# For PRs, checkout the base branch to compare against
- name: Checkout base branch
if: github.base_ref
uses: actions/checkout@v4
with:
ref: ${{ github.base_ref }}
clean: true

- name: Clean Docker system
run: |
docker system prune -af
docker builder prune -af
- name: Build base branch image
if: github.base_ref
id: base_build_time
run: |
start_time=$(date +%s)
DOCKER_BUILDKIT=1 docker build . --no-cache --progress=plain
end_time=$(date +%s)
base_time=$((end_time - start_time))
echo "base_time=$base_time" >> $GITHUB_OUTPUT
# For PRs, checkout back to the PR's HEAD
- name: Checkout PR branch
if: github.base_ref
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
clean: true

- name: Clean Docker system again
run: |
docker system prune -af
docker builder prune -af
- name: Build and time Docker image
id: build_time
run: |
start_time=$(date +%s)
DOCKER_BUILDKIT=1 docker build . --no-cache --progress=plain
end_time=$(date +%s)
build_time=$((end_time - start_time))
echo "build_time=$build_time" >> $GITHUB_OUTPUT
- name: Compare build times
run: |
CURRENT_TIME=${{ steps.build_time.outputs.build_time }}
if [ "${{ github.base_ref }}" != "" ]; then
BASE_TIME=${{ steps.base_build_time.outputs.base_time }}
echo "### 🏗️ Docker Build Time Comparison" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Branch | Build Time | Difference |" >> $GITHUB_STEP_SUMMARY
echo "|--------|------------|------------|" >> $GITHUB_STEP_SUMMARY
echo "| base (${{ github.base_ref }}) | ${BASE_TIME}s | - |" >> $GITHUB_STEP_SUMMARY
DIFF=$((CURRENT_TIME - BASE_TIME))
PERCENT=$(echo "scale=2; ($CURRENT_TIME - $BASE_TIME) * 100 / $BASE_TIME" | bc)
if [ $DIFF -gt 0 ]; then
DIFF_TEXT="⬆️ +${DIFF}s (+${PERCENT}%)"
elif [ $DIFF -lt 0 ]; then
DIFF_TEXT="⬇️ ${DIFF}s (${PERCENT}%)"
else
DIFF_TEXT="✨ No change"
fi
echo "| current (${{ github.head_ref }}) | ${CURRENT_TIME}s | $DIFF_TEXT |" >> $GITHUB_STEP_SUMMARY
# Fail if build time increased by more than 5%
if (( $(echo "$PERCENT > 5" | bc -l) )); then
echo "" >> $GITHUB_STEP_SUMMARY
echo "❌ **Build time increased by more than 5%!**" >> $GITHUB_STEP_SUMMARY
echo "This change significantly increases the build time. Please review the Dockerfile changes." >> $GITHUB_STEP_SUMMARY
exit 1
elif (( $(echo "$PERCENT < 0" | bc -l) )); then
echo "" >> $GITHUB_STEP_SUMMARY
echo "✅ **Build time decreased!**" >> $GITHUB_STEP_SUMMARY
echo "Great job optimizing the build!" >> $GITHUB_STEP_SUMMARY
fi
else
echo "### 🏗️ Docker Build Time" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Build completed in ${CURRENT_TIME} seconds" >> $GITHUB_STEP_SUMMARY
fi
6 changes: 4 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ src/prefect/server/ui_build/*

# API artifacts


# MacOS
.DS_Store

Expand All @@ -76,4 +75,7 @@ libcairo.2.dylib

# setuptools-scm generated files
src/integrations/*/**/_version.py
*.log
*.log

# Pyright type analysis report
prefect-analysis.json
2 changes: 1 addition & 1 deletion .nvmrc
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v18.18.0
18.18.0
Loading

0 comments on commit 39bf7bf

Please sign in to comment.