From 813c9974a3db20ed6c6de1d3b60408f367e7c4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ladislav=20G=C3=A1l?= <129292521+GalLadislav@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:21:22 +0100 Subject: [PATCH 1/4] Enable PREFECT_PROFILES_PATH in source files (#15914) --- src/prefect/settings/sources.py | 21 ++++ tests/test_settings.py | 176 ++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) diff --git a/src/prefect/settings/sources.py b/src/prefect/settings/sources.py index 98e1d9c810a6..bef5a138e904 100644 --- a/src/prefect/settings/sources.py +++ b/src/prefect/settings/sources.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Type +import dotenv import toml from pydantic import AliasChoices from pydantic.fields import FieldInfo @@ -15,6 +16,7 @@ from pydantic_settings.sources import ConfigFileSourceMixin from prefect.settings.constants import DEFAULT_PREFECT_HOME, DEFAULT_PROFILES_PATH +from prefect.utilities.collections import get_from_dict class EnvFilterSettingsSource(EnvSettingsSource): @@ -230,6 +232,25 @@ def _get_profiles_path() -> Path: return DEFAULT_PROFILES_PATH if env_path := os.getenv("PREFECT_PROFILES_PATH"): return Path(env_path) + if dotenv_path := dotenv.dotenv_values(".env").get("PREFECT_PROFILES_PATH"): + return Path(dotenv_path) + if toml_path := _get_profiles_path_from_toml("prefect.toml", ["profiles_path"]): + return Path(toml_path) + if pyproject_path := _get_profiles_path_from_toml( + "pyproject.toml", ["tool", "prefect", "profiles_path"] + ): + return Path(pyproject_path) if not (DEFAULT_PREFECT_HOME / "profiles.toml").exists(): return DEFAULT_PROFILES_PATH return DEFAULT_PREFECT_HOME / "profiles.toml" + + +def _get_profiles_path_from_toml(path: str, keys: List[str]) -> Optional[str]: + """Helper to get the profiles path from a toml file.""" + + try: + toml_data = toml.load(path) + except FileNotFoundError: + return None + + return get_from_dict(toml_data, keys) diff --git a/tests/test_settings.py b/tests/test_settings.py index cbf877dd9d3a..8c3432e784d4 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -461,8 +461,10 @@ def temporary_toml_file(tmp_path): toml_file = Path("prefect.toml") def _create_temp_toml(content, path=toml_file): + nonlocal toml_file with path.open("w") as f: toml.dump(content, f) + toml_file = path # update toml_file in case path was changed yield _create_temp_toml @@ -1213,6 +1215,180 @@ def test_resolution_order_with_nested_settings( assert Settings().api.url == "http://example.com:4200" + def test_profiles_path_from_env_source( + self, temporary_env_file, monkeypatch, tmp_path + ): + profiles_path = tmp_path / "custom_profiles.toml" + + monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) + monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) + + profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "420,500" + """ + ) + ) + + temporary_env_file(f"PREFECT_PROFILES_PATH={profiles_path}") + + assert Settings().profiles_path == profiles_path + assert Settings().client.retry_extra_codes == {420, 500} + + os.unlink(".env") + + monkeypatch.setenv("PREFECT_TEST_MODE", "1") + monkeypatch.setenv("PREFECT_TESTING_UNIT_TEST_MODE", "1") + + assert Settings().client.retry_extra_codes == set() + + def test_profiles_path_from_toml_source( + self, temporary_toml_file, monkeypatch, tmp_path + ): + profiles_path = tmp_path / "custom_profiles.toml" + + monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) + monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) + + profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "420,500" + """ + ) + ) + + temporary_toml_file({"profiles_path": str(profiles_path)}) + + assert Settings().profiles_path == profiles_path + assert Settings().client.retry_extra_codes == {420, 500} + + os.unlink("prefect.toml") + + monkeypatch.setenv("PREFECT_TEST_MODE", "1") + monkeypatch.setenv("PREFECT_TESTING_UNIT_TEST_MODE", "1") + + assert Settings().client.retry_extra_codes == set() + + def test_profiles_path_from_pyproject_source( + self, temporary_toml_file, monkeypatch, tmp_path + ): + monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) + monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) + + profiles_path = tmp_path / "custom_profiles.toml" + profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "420,500" + """ + ) + ) + + temporary_toml_file( + {"tool": {"prefect": {"profiles_path": str(profiles_path)}}}, + path=Path("pyproject.toml"), + ) + + assert Settings().profiles_path == profiles_path + assert Settings().client.retry_extra_codes == {420, 500} + + os.unlink("pyproject.toml") + + monkeypatch.setenv("PREFECT_TEST_MODE", "1") + monkeypatch.setenv("PREFECT_TESTING_UNIT_TEST_MODE", "1") + + assert Settings().client.retry_extra_codes == set() + + def test_profiles_path_resolution_order_from_sources( + self, temporary_env_file, monkeypatch, tmp_path + ): + monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) + monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) + + pyproject_profiles_path = tmp_path / "pyproject_profiles.toml" + pyproject_profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "420,500" + """ + ) + ) + + toml_profiles_path = tmp_path / "toml_profiles.toml" + toml_profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "300" + """ + ) + ) + + env_profiles_path = tmp_path / "env_profiles.toml" + env_profiles_path.write_text( + textwrap.dedent( + """ + active = "foo" + + [profiles.foo] + PREFECT_CLIENT_RETRY_EXTRA_CODES = "200" + """ + ) + ) + + with open("pyproject.toml", "w") as f: + toml.dump( + {"tool": {"prefect": {"profiles_path": str(pyproject_profiles_path)}}}, + f, + ) + + assert Settings().profiles_path == pyproject_profiles_path + assert Settings().client.retry_extra_codes == {420, 500} + + with open("prefect.toml", "w") as f: + toml.dump({"profiles_path": str(toml_profiles_path)}, f) + + assert Settings().profiles_path == toml_profiles_path + assert Settings().client.retry_extra_codes == {300} + + temporary_env_file(f"PREFECT_PROFILES_PATH={env_profiles_path}") + + assert Settings().profiles_path == env_profiles_path + assert Settings().client.retry_extra_codes == {200} + + os.unlink(".env") + + assert Settings().profiles_path == toml_profiles_path + assert Settings().client.retry_extra_codes == {300} + + os.unlink("prefect.toml") + + assert Settings().profiles_path == pyproject_profiles_path + assert Settings().client.retry_extra_codes == {420, 500} + + os.unlink("pyproject.toml") + + monkeypatch.setenv("PREFECT_TEST_MODE", "1") + monkeypatch.setenv("PREFECT_TESTING_UNIT_TEST_MODE", "1") + + assert Settings().client.retry_extra_codes == set() + class TestLoadProfiles: @pytest.fixture(autouse=True) From 53ce5c2b72f86880cc5e65f9d779591de386b0dc Mon Sep 17 00:00:00 2001 From: nate nowack Date: Wed, 6 Nov 2024 09:17:07 -0600 Subject: [PATCH 2/4] fix sqlite migration (#15917) --- .github/workflows/integration-tests.yaml | 41 +++++++++++-- old-sqlite.Dockerfile | 59 +++++++++++++++++++ ...d4ad99c_rename_block_to_blockbasis_and_.py | 21 ++++++- ...e_add_deployment_to_global_concurrency_.py | 23 ++++---- 4 files changed, 126 insertions(+), 18 deletions(-) create mode 100644 old-sqlite.Dockerfile diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1a3ff9971ced..ab3c069447ab 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -1,3 +1,6 @@ +# TODO: Replace `wait-for-server` with dedicated command +# https://github.com/PrefectHQ/prefect/issues/6990 + name: Integration tests on: pull_request: @@ -85,9 +88,6 @@ jobs: ./scripts/wait-for-server.py - # TODO: Replace `wait-for-server` with dedicated command - # https://github.com/PrefectHQ/prefect/issues/6990 - - name: Start server if: ${{ matrix.server-version.version == 'main' }} env: @@ -98,9 +98,6 @@ jobs: ./scripts/wait-for-server.py - # TODO: Replace `wait-for-server` with dedicated command - # https://github.com/PrefectHQ/prefect/issues/6990 - - name: Run integration flows env: PREFECT_API_URL: http://127.0.0.1:4200/api @@ -113,3 +110,35 @@ jobs: run: | cat server.log || echo "No logs available" docker logs prefect-server || echo "No logs available" + + sqlite-3-24-0: + name: Test SQLite 3.24.0 Compatibility + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Test with SQLite 3.24.0 + run: > + docker build -t prefect-server-old-sqlite \ + --build-arg SQLITE_VERSION=3240000 \ + --build-arg SQLITE_YEAR=2018 \ + -f old-sqlite.Dockerfile . && + docker run prefect-server-old-sqlite sh -c "prefect server database downgrade --yes -r base && prefect server database upgrade --yes" + + sqlite-3-31-1: + name: Test SQLite 3.31.1 Compatibility + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Test with SQLite 3.31.1 + run: > + docker build -t prefect-server-new-sqlite \ + --build-arg SQLITE_VERSION=3310100 \ + --build-arg SQLITE_YEAR=2020 \ + -f old-sqlite.Dockerfile . && + docker run prefect-server-new-sqlite sh -c "prefect server database downgrade --yes -r base && prefect server database upgrade --yes" diff --git a/old-sqlite.Dockerfile b/old-sqlite.Dockerfile new file mode 100644 index 000000000000..23a85f34e977 --- /dev/null +++ b/old-sqlite.Dockerfile @@ -0,0 +1,59 @@ +# Build the Python distributable +FROM python:3.9-slim AS python-builder + +WORKDIR /opt/prefect + +# Install git for version calculation +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Copy the repository for version calculation +COPY . . + +# Create source distribution +RUN python setup.py sdist && \ + mv "dist/$(python setup.py --fullname).tar.gz" "dist/prefect.tar.gz" + +# Final image +FROM python:3.9-slim + +# Accept SQLite version as build argument +ARG SQLITE_VERSION="3310100" +ARG SQLITE_YEAR="2020" + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + wget + +# Download and compile SQLite +RUN wget https://www.sqlite.org/${SQLITE_YEAR}/sqlite-autoconf-${SQLITE_VERSION}.tar.gz \ + && tar xvfz sqlite-autoconf-${SQLITE_VERSION}.tar.gz \ + && cd sqlite-autoconf-${SQLITE_VERSION} \ + && ./configure \ + && make \ + && make install \ + && ldconfig \ + && cd .. \ + && rm -rf sqlite-autoconf-${SQLITE_VERSION}* + +# Install uv for faster pip operations +COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/uv +ENV UV_SYSTEM_PYTHON=1 + +# Set library path to use our compiled SQLite +ENV LD_LIBRARY_PATH=/usr/local/lib + +WORKDIR /app + +# Copy the built distributable +COPY --from=python-builder /opt/prefect/dist/prefect.tar.gz ./dist/ + +# Install requirements and Prefect +COPY requirements*.txt ./ +RUN uv pip install -r requirements.txt +RUN uv pip install ./dist/prefect.tar.gz + + diff --git a/src/prefect/server/database/migrations/versions/sqlite/2022_04_23_114831_fd966d4ad99c_rename_block_to_blockbasis_and_.py b/src/prefect/server/database/migrations/versions/sqlite/2022_04_23_114831_fd966d4ad99c_rename_block_to_blockbasis_and_.py index a80a2d91514f..beb48e8ec5ec 100644 --- a/src/prefect/server/database/migrations/versions/sqlite/2022_04_23_114831_fd966d4ad99c_rename_block_to_blockbasis_and_.py +++ b/src/prefect/server/database/migrations/versions/sqlite/2022_04_23_114831_fd966d4ad99c_rename_block_to_blockbasis_and_.py @@ -16,22 +16,30 @@ def upgrade(): + # First drop the foreign key constraints + with op.batch_alter_table("block", schema=None) as batch_op: + batch_op.drop_constraint("fk_block__block_spec_id__block_spec") + + # Then rename the tables op.rename_table("block_spec", "block_schema") op.rename_table("block", "block_document") + # Handle indexes and column renames for block_document with op.batch_alter_table("block_document", schema=None) as batch_op: + # Drop indexes first batch_op.drop_index("ix_block__is_default_storage_block") batch_op.drop_index("ix_block__name") batch_op.drop_index("ix_block__updated") batch_op.drop_index("uq_block__spec_id_name") + + # Rename columns batch_op.alter_column("block_spec_id", new_column_name="block_schema_id") batch_op.alter_column( "is_default_storage_block", new_column_name="is_default_storage_block_document", ) - batch_op.drop_constraint("fk_block__block_spec_id__block_spec") - batch_op.drop_constraint("pk_block_data") + # Create new indexes with op.batch_alter_table("block_document", schema=None) as batch_op: batch_op.create_index( batch_op.f("ix_block_document__is_default_storage_block_document"), @@ -48,6 +56,15 @@ def upgrade(): "uq_block__schema_id_name", ["block_schema_id", "name"], unique=True ) + # Re-create foreign key at the end + batch_op.create_foreign_key( + batch_op.f("fk_block__block_schema_id__block_schema"), + "block_schema", + ["block_schema_id"], + ["id"], + ondelete="cascade", + ) + with op.batch_alter_table("block_schema", schema=None) as batch_op: batch_op.drop_index("ix_block_spec__type") batch_op.drop_index("ix_block_spec__updated") diff --git a/src/prefect/server/database/migrations/versions/sqlite/2024_09_16_162719_4ad4658cbefe_add_deployment_to_global_concurrency_.py b/src/prefect/server/database/migrations/versions/sqlite/2024_09_16_162719_4ad4658cbefe_add_deployment_to_global_concurrency_.py index bd3a6c197ebb..edb13553a91f 100644 --- a/src/prefect/server/database/migrations/versions/sqlite/2024_09_16_162719_4ad4658cbefe_add_deployment_to_global_concurrency_.py +++ b/src/prefect/server/database/migrations/versions/sqlite/2024_09_16_162719_4ad4658cbefe_add_deployment_to_global_concurrency_.py @@ -5,6 +5,7 @@ Create Date: 2024-09-16 16:27:19.451150 """ + import sqlalchemy as sa from alembic import op @@ -37,16 +38,18 @@ def upgrade(): # migrate existing data sql = sa.text( """ - WITH deployment_limit_mapping AS ( - SELECT d.id AS deployment_id, l.id AS limit_id - FROM deployment d - JOIN concurrency_limit_v2 l ON l.name = 'deployment:' || d.id - ) - UPDATE deployment - SET concurrency_limit_id = dlm.limit_id - FROM deployment_limit_mapping dlm - WHERE deployment.id = dlm.deployment_id; - """ + UPDATE deployment + SET concurrency_limit_id = ( + SELECT l.id + FROM concurrency_limit_v2 l + WHERE l.name = 'deployment:' || deployment.id + ) + WHERE EXISTS ( + SELECT 1 + FROM concurrency_limit_v2 l + WHERE l.name = 'deployment:' || deployment.id + ); + """ ) op.execute(sql) From 5f5427c410cd04505d7b2c701e2003f856044178 Mon Sep 17 00:00:00 2001 From: Prefect <41086007+marvin-robot@users.noreply.github.com> Date: Wed, 6 Nov 2024 12:35:38 -0600 Subject: [PATCH 3/4] Update @prefecthq/prefect-design to version 2.14.5 (#15934) Co-authored-by: marvin-robot --- ui/package-lock.json | 14 +++++++------- ui/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index be1046a0c4eb..146954272a9d 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,7 +8,7 @@ "name": "@prefecthq/ui", "version": "2.8.0", "dependencies": { - "@prefecthq/prefect-design": "2.14.4", + "@prefecthq/prefect-design": "2.14.5", "@prefecthq/prefect-ui-library": "3.11.14", "@prefecthq/vue-charts": "2.0.5", "@prefecthq/vue-compositions": "1.11.5", @@ -1037,9 +1037,9 @@ } }, "node_modules/@prefecthq/prefect-design": { - "version": "2.14.4", - "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.4.tgz", - "integrity": "sha512-jk6C4eYVT6GYTMVeZWalkGB8ckwg7/XZhIYmsw80kl8jT5a7pB7jqhMi6ijYEdJG2P4lJTgWKX7qV0MXfARkBg==", + "version": "2.14.5", + "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.5.tgz", + "integrity": "sha512-Xj/F0QgZhY/xGMkxis4++x8E0HqxL2IrZJxNDD0k1zhzVX3/hTgSG5cbuZ9I5e6ZbA9IfVhkKvHUKvzYuriMYQ==", "dependencies": { "@fontsource-variable/inconsolata": "^5.0.18", "@fontsource-variable/inter": "^5.0.18", @@ -7627,9 +7627,9 @@ } }, "@prefecthq/prefect-design": { - "version": "2.14.4", - "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.4.tgz", - "integrity": "sha512-jk6C4eYVT6GYTMVeZWalkGB8ckwg7/XZhIYmsw80kl8jT5a7pB7jqhMi6ijYEdJG2P4lJTgWKX7qV0MXfARkBg==", + "version": "2.14.5", + "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.5.tgz", + "integrity": "sha512-Xj/F0QgZhY/xGMkxis4++x8E0HqxL2IrZJxNDD0k1zhzVX3/hTgSG5cbuZ9I5e6ZbA9IfVhkKvHUKvzYuriMYQ==", "requires": { "@fontsource-variable/inconsolata": "^5.0.18", "@fontsource-variable/inter": "^5.0.18", diff --git a/ui/package.json b/ui/package.json index b191871e2166..c8df6e9547a5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -10,7 +10,7 @@ "validate:types": "vue-tsc --noEmit" }, "dependencies": { - "@prefecthq/prefect-design": "2.14.4", + "@prefecthq/prefect-design": "2.14.5", "@prefecthq/prefect-ui-library": "3.11.14", "@prefecthq/vue-charts": "2.0.5", "@prefecthq/vue-compositions": "1.11.5", From 575a1f6fe9f2bbb683aa017f2b519a27bf913a58 Mon Sep 17 00:00:00 2001 From: Prefect <41086007+marvin-robot@users.noreply.github.com> Date: Wed, 6 Nov 2024 15:28:52 -0600 Subject: [PATCH 4/4] Update @prefecthq/prefect-design to version 2.14.6 (#15937) Co-authored-by: marvin-robot --- ui/package-lock.json | 14 +++++++------- ui/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/package-lock.json b/ui/package-lock.json index 146954272a9d..c2b962951e61 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,7 +8,7 @@ "name": "@prefecthq/ui", "version": "2.8.0", "dependencies": { - "@prefecthq/prefect-design": "2.14.5", + "@prefecthq/prefect-design": "2.14.6", "@prefecthq/prefect-ui-library": "3.11.14", "@prefecthq/vue-charts": "2.0.5", "@prefecthq/vue-compositions": "1.11.5", @@ -1037,9 +1037,9 @@ } }, "node_modules/@prefecthq/prefect-design": { - "version": "2.14.5", - "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.5.tgz", - "integrity": "sha512-Xj/F0QgZhY/xGMkxis4++x8E0HqxL2IrZJxNDD0k1zhzVX3/hTgSG5cbuZ9I5e6ZbA9IfVhkKvHUKvzYuriMYQ==", + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.6.tgz", + "integrity": "sha512-iHFJ6G4pTbLIQtCfvduJoByG1aGRwNVgXmcdGoy2vGPw/dX+VsZLf1XMOkhXmkCGFsjNrqP5fddJ+xIgqahrVA==", "dependencies": { "@fontsource-variable/inconsolata": "^5.0.18", "@fontsource-variable/inter": "^5.0.18", @@ -7627,9 +7627,9 @@ } }, "@prefecthq/prefect-design": { - "version": "2.14.5", - "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.5.tgz", - "integrity": "sha512-Xj/F0QgZhY/xGMkxis4++x8E0HqxL2IrZJxNDD0k1zhzVX3/hTgSG5cbuZ9I5e6ZbA9IfVhkKvHUKvzYuriMYQ==", + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/@prefecthq/prefect-design/-/prefect-design-2.14.6.tgz", + "integrity": "sha512-iHFJ6G4pTbLIQtCfvduJoByG1aGRwNVgXmcdGoy2vGPw/dX+VsZLf1XMOkhXmkCGFsjNrqP5fddJ+xIgqahrVA==", "requires": { "@fontsource-variable/inconsolata": "^5.0.18", "@fontsource-variable/inter": "^5.0.18", diff --git a/ui/package.json b/ui/package.json index c8df6e9547a5..19c3554c9b10 100644 --- a/ui/package.json +++ b/ui/package.json @@ -10,7 +10,7 @@ "validate:types": "vue-tsc --noEmit" }, "dependencies": { - "@prefecthq/prefect-design": "2.14.5", + "@prefecthq/prefect-design": "2.14.6", "@prefecthq/prefect-ui-library": "3.11.14", "@prefecthq/vue-charts": "2.0.5", "@prefecthq/vue-compositions": "1.11.5",