Skip to content

Commit

Permalink
update entrypoint.sh to use uv for EXTRA_PIP_PACKAGES (#16291)
Browse files Browse the repository at this point in the history
  • Loading branch information
zzstoatzz authored Dec 12, 2024
1 parent 1d6ae61 commit 547d022
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 3 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/python-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ on:
- requirements-dev.txt
- setup.cfg
- Dockerfile
- scripts/entrypoint.sh
push:
branches:
- main
Expand All @@ -28,7 +29,7 @@ on:
- requirements-dev.txt
- setup.cfg
- Dockerfile

- scripts/entrypoint.sh
permissions:
contents: read
actions: write
Expand Down
105 changes: 105 additions & 0 deletions flows/docker_deploy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import asyncio
import subprocess
import sys
from pathlib import Path
from textwrap import dedent

from prefect import flow, get_client
from prefect.deployments import run_deployment
from prefect.docker.docker_image import DockerImage


async def read_flow_run(flow_run_id: str):
"""Read a flow run's state using the Prefect client."""
async with get_client() as client:
return await client.read_flow_run(flow_run_id)


@flow
def flow_that_needs_pandas() -> str:
"""A flow that needs pandas."""
import pandas

df = pandas.DataFrame({"a": [1, 2, 3]})

assert isinstance(df, pandas.DataFrame)

return "we're done"


def main():
"""
Deploy and run a flow in a Docker container with runtime package installation using uv.
Demonstrates using EXTRA_PIP_PACKAGES to install dependencies at runtime.
"""
try:
subprocess.check_call(
["prefect", "work-pool", "create", "test-docker-pool", "-t", "docker"],
stdout=sys.stdout,
stderr=sys.stderr,
)

dockerfile = Path("docker-deploy.Dockerfile")

dockerfile.write_text(
dedent(
"""
FROM prefecthq/prefect:3-latest
COPY flows/docker_deploy.py /opt/prefect/flows/docker_deploy.py
"""
)
)

flow_that_needs_pandas.deploy(
name="docker-demo-deployment",
work_pool_name="test-docker-pool",
job_variables={
"env": {"EXTRA_PIP_PACKAGES": "pandas"},
"image": "prefect-integration-test-docker-deploy",
},
image=DockerImage(
name="prefect-integration-test-docker-deploy",
dockerfile=str(dockerfile),
),
build=True,
push=False,
)

dockerfile.unlink()

flow_run = run_deployment(
"flow-that-needs-pandas/docker-demo-deployment",
timeout=0,
)

# Execute the flow run
subprocess.check_call(
[
"prefect",
"worker",
"start",
"--pool",
"test-docker-pool",
"--run-once",
"--install-policy",
"if-not-present",
],
stdout=sys.stdout,
stderr=sys.stderr,
)

# Check the flow run state
flow_run = asyncio.run(read_flow_run(flow_run.id))
assert flow_run.state.is_completed(), flow_run.state

finally:
# Cleanup
subprocess.check_call(
["prefect", "--no-prompt", "work-pool", "delete", "test-docker-pool"],
stdout=sys.stdout,
stderr=sys.stderr,
)


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions scripts/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ if [ -f ~/.bashrc ]; then
fi

if [ ! -z "$EXTRA_PIP_PACKAGES" ]; then
echo "+pip install $EXTRA_PIP_PACKAGES"
pip install $EXTRA_PIP_PACKAGES
echo "+uv pip install $EXTRA_PIP_PACKAGES"
uv pip install --system $EXTRA_PIP_PACKAGES
fi

if [ -z "$*" ]; then
Expand Down

0 comments on commit 547d022

Please sign in to comment.