diff --git a/examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py b/examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py index e0def4fcf16ae..d70ef1b256411 100644 --- a/examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py +++ b/examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py @@ -1,6 +1,6 @@ import sys -from dagster_external import init_dagster_external +from dagster_externals import init_dagster_externals class SomeSqlClient: @@ -11,7 +11,7 @@ def query(self, query_str: str) -> None: if __name__ == "__main__": sql = sys.argv[1] - context = init_dagster_external() + context = init_dagster_externals() client = SomeSqlClient() client.query(sql) diff --git a/examples/experimental/assets_yaml_dsl/setup.py b/examples/experimental/assets_yaml_dsl/setup.py index e758d92bc4611..66935dd0dcd4b 100644 --- a/examples/experimental/assets_yaml_dsl/setup.py +++ b/examples/experimental/assets_yaml_dsl/setup.py @@ -3,7 +3,7 @@ setup( name="assets_yaml_dsl", packages=find_packages(exclude=["assets_yaml_dsl_tests"]), - install_requires=["dagster", "pandas", "dagster_external"], + install_requires=["dagster", "pandas", "dagster_externals"], license="Apache-2.0", description="Dagster example of yaml dsl for building asset graphs", url="https://github.com/dagster-io/dagster/tree/master/examples/assets_yaml_dsl", diff --git a/examples/experimental/assets_yaml_dsl/tox.ini b/examples/experimental/assets_yaml_dsl/tox.ini index 59217fc026a25..b358782d698f8 100644 --- a/examples/experimental/assets_yaml_dsl/tox.ini +++ b/examples/experimental/assets_yaml_dsl/tox.ini @@ -6,7 +6,7 @@ download = True passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* deps = -e ../../../python_modules/dagster[test] - -e ../../../python_modules/dagster-external + -e ../../../python_modules/dagster-externals -e ../../../python_modules/dagster-webserver -e ../../../python_modules/dagster-graphql -e . diff --git a/pyright/master/requirements-pinned.txt b/pyright/master/requirements-pinned.txt index 1ea2ef331d15b..54012f9c5a630 100644 --- a/pyright/master/requirements-pinned.txt +++ b/pyright/master/requirements-pinned.txt @@ -90,7 +90,7 @@ cryptography==41.0.2 cycler==0.11.0 -e python_modules/dagit -e python_modules/dagster --e python_modules/dagster-external +-e python_modules/dagster-externals -e python_modules/libraries/dagster-airbyte -e python_modules/libraries/dagster-airflow -e python_modules/libraries/dagster-aws diff --git a/pyright/master/requirements.txt b/pyright/master/requirements.txt index 5eeb6829596c4..562d1c584807f 100644 --- a/pyright/master/requirements.txt +++ b/pyright/master/requirements.txt @@ -31,7 +31,7 @@ -e python_modules/dagit -e python_modules/dagster-graphql -e python_modules/dagster-test --e python_modules/dagster-external +-e python_modules/dagster-externals -e python_modules/dagster-webserver -e python_modules/libraries/dagster-airbyte/ -e python_modules/libraries/dagster-airflow[test_airflow_2] diff --git a/python_modules/dagster-external/README.md b/python_modules/dagster-external/README.md deleted file mode 100644 index 9b2e1cd882812..0000000000000 --- a/python_modules/dagster-external/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# dagster-external - -The docs for `dagster-external ` can be found -[here](https://docs.dagster.io/_apidocs/libraries/dagster-external). \ No newline at end of file diff --git a/python_modules/dagster-external/dagster_external/__init__.py b/python_modules/dagster-external/dagster_external/__init__.py deleted file mode 100644 index 58adec48a996d..0000000000000 --- a/python_modules/dagster-external/dagster_external/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from dagster_external.context import ( - ExternalExecutionContext as ExternalExecutionContext, - init_dagster_external as init_dagster_external, - is_dagster_orchestration_active as is_dagster_orchestration_active, -) -from dagster_external.protocol import ( - ExternalExecutionContextData as ExternalExecutionContextData, -) diff --git a/python_modules/dagster-external/dagster_external_tests/test_version.py b/python_modules/dagster-external/dagster_external_tests/test_version.py deleted file mode 100644 index 5275d7c9598d7..0000000000000 --- a/python_modules/dagster-external/dagster_external_tests/test_version.py +++ /dev/null @@ -1,5 +0,0 @@ -from dagster_external.version import __version__ - - -def test_version(): - assert __version__ diff --git a/python_modules/dagster-external/.coveragerc b/python_modules/dagster-externals/.coveragerc similarity index 100% rename from python_modules/dagster-external/.coveragerc rename to python_modules/dagster-externals/.coveragerc diff --git a/python_modules/dagster-external/LICENSE b/python_modules/dagster-externals/LICENSE similarity index 100% rename from python_modules/dagster-external/LICENSE rename to python_modules/dagster-externals/LICENSE diff --git a/python_modules/dagster-external/MANIFEST.in b/python_modules/dagster-externals/MANIFEST.in similarity index 100% rename from python_modules/dagster-external/MANIFEST.in rename to python_modules/dagster-externals/MANIFEST.in diff --git a/python_modules/dagster-externals/README.md b/python_modules/dagster-externals/README.md new file mode 100644 index 0000000000000..357bf1a270bc1 --- /dev/null +++ b/python_modules/dagster-externals/README.md @@ -0,0 +1,4 @@ +# dagster-externals + +The docs for `dagster-externals ` can be found +[here](https://docs.dagster.io/_apidocs/libraries/dagster-externals). diff --git a/python_modules/dagster-externals/dagster_externals/__init__.py b/python_modules/dagster-externals/dagster_externals/__init__.py new file mode 100644 index 0000000000000..b2338f83ba976 --- /dev/null +++ b/python_modules/dagster-externals/dagster_externals/__init__.py @@ -0,0 +1,19 @@ +from dagster_externals._context import ( + ExternalExecutionContext as ExternalExecutionContext, + init_dagster_externals as init_dagster_externals, + is_dagster_orchestration_active as is_dagster_orchestration_active, +) +from dagster_externals._protocol import ( + DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME, + DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME, + DAGSTER_EXTERNALS_ENV_KEYS as DAGSTER_EXTERNALS_ENV_KEYS, + ExternalDataProvenance as ExternalDataProvenance, + ExternalExecutionContextData as ExternalExecutionContextData, + ExternalExecutionExtras as ExternalExecutionExtras, + ExternalPartitionKeyRange as ExternalPartitionKeyRange, + ExternalTimeWindow as ExternalTimeWindow, + Notification as Notification, +) +from dagster_externals._util import ( + DagsterExternalError as DagsterExternalError, +) diff --git a/python_modules/dagster-external/dagster_external/context.py b/python_modules/dagster-externals/dagster_externals/_context.py similarity index 94% rename from python_modules/dagster-external/dagster_external/context.py rename to python_modules/dagster-externals/dagster_externals/_context.py index 80bdbe3cd3ac5..4bd667b44fe10 100644 --- a/python_modules/dagster-external/dagster_external/context.py +++ b/python_modules/dagster-externals/dagster_externals/_context.py @@ -6,17 +6,16 @@ from typing_extensions import Self -from dagster_external.params import get_external_execution_params - -from .protocol import ( - DAGSTER_EXTERNAL_ENV_KEYS, +from ._params import get_external_execution_params +from ._protocol import ( + DAGSTER_EXTERNALS_ENV_KEYS, ExternalDataProvenance, ExternalExecutionContextData, ExternalPartitionKeyRange, ExternalTimeWindow, Notification, ) -from .util import ( +from ._util import ( assert_defined_asset_property, assert_defined_extra, assert_defined_partition_property, @@ -28,10 +27,10 @@ def is_dagster_orchestration_active() -> bool: - return bool(os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["is_orchestration_active"])) + return bool(os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["is_orchestration_active"])) -def init_dagster_external() -> "ExternalExecutionContext": +def init_dagster_externals() -> "ExternalExecutionContext": if ExternalExecutionContext.is_initialized(): return ExternalExecutionContext.get() @@ -46,7 +45,7 @@ def init_dagster_external() -> "ExternalExecutionContext": warnings.warn( "This process was not launched by a Dagster orchestration process. All calls to the" - " `dagster-external` context are no-ops." + " `dagster-externals` context are no-ops." ) context = MagicMock() ExternalExecutionContext.set(context) @@ -82,7 +81,7 @@ def get(cls) -> Self: if cls._instance is None: raise Exception( "ExternalExecutionContext has not been initialized. You must call" - " `init_dagster_external()`." + " `init_dagster_externals()`." ) return cls._instance diff --git a/python_modules/dagster-external/dagster_external/params.py b/python_modules/dagster-externals/dagster_externals/_params.py similarity index 66% rename from python_modules/dagster-external/dagster_external/params.py rename to python_modules/dagster-externals/dagster_externals/_params.py index 6ec50037a9bc9..3906fbd4bf810 100644 --- a/python_modules/dagster-external/dagster_external/params.py +++ b/python_modules/dagster-externals/dagster_externals/_params.py @@ -1,14 +1,14 @@ import os from dataclasses import dataclass -from dagster_external.protocol import ( - DAGSTER_EXTERNAL_ENV_KEYS, +from ._protocol import ( + DAGSTER_EXTERNALS_ENV_KEYS, ) def get_external_execution_params() -> "ExternalExecutionParams": - input_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["input"]) - output_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["output"]) + input_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["input"]) + output_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["output"]) assert input_path, "input_path must be set" assert output_path, "output_path must be set" return ExternalExecutionParams( diff --git a/python_modules/dagster-external/dagster_external/protocol.py b/python_modules/dagster-externals/dagster_externals/_protocol.py similarity index 76% rename from python_modules/dagster-external/dagster_external/protocol.py rename to python_modules/dagster-externals/dagster_externals/_protocol.py index dedcc64092224..bb5b2def38437 100644 --- a/python_modules/dagster-external/dagster_external/protocol.py +++ b/python_modules/dagster-externals/dagster_externals/_protocol.py @@ -6,13 +6,13 @@ # ##### PARAMETERS -DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input" -DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output" +DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input" +DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output" -DAGSTER_EXTERNAL_ENV_KEYS: Final = { - "is_orchestration_active": "DAGSTER_EXTERNAL_IS_ORCHESTRATION_ACTIVE", - "input": "DAGSTER_EXTERNAL_INPUT", - "output": "DAGSTER_EXTERNAL_OUTPUT", +DAGSTER_EXTERNALS_ENV_KEYS: Final = { + "is_orchestration_active": "DAGSTER_EXTERNALS_IS_ORCHESTRATION_ACTIVE", + "input": "DAGSTER_EXTERNALS_INPUT", + "output": "DAGSTER_EXTERNALS_OUTPUT", } # ##### NOTIFICATION diff --git a/python_modules/dagster-external/dagster_external/util.py b/python_modules/dagster-externals/dagster_externals/_util.py similarity index 96% rename from python_modules/dagster-external/dagster_external/util.py rename to python_modules/dagster-externals/dagster_externals/_util.py index 25bf88aedf71a..d1cc877690724 100644 --- a/python_modules/dagster-external/dagster_external/util.py +++ b/python_modules/dagster-externals/dagster_externals/_util.py @@ -1,7 +1,7 @@ import json from typing import Any, Optional, Sequence, TypeVar -from dagster_external.protocol import ExternalExecutionContextData, ExternalExecutionExtras +from ._protocol import ExternalExecutionContextData, ExternalExecutionExtras T = TypeVar("T") diff --git a/python_modules/dagster-external/dagster_external/py.typed b/python_modules/dagster-externals/dagster_externals/py.typed similarity index 100% rename from python_modules/dagster-external/dagster_external/py.typed rename to python_modules/dagster-externals/dagster_externals/py.typed diff --git a/python_modules/dagster-external/dagster_external/version.py b/python_modules/dagster-externals/dagster_externals/version.py similarity index 100% rename from python_modules/dagster-external/dagster_external/version.py rename to python_modules/dagster-externals/dagster_externals/version.py diff --git a/python_modules/dagster-external/dagster_external_tests/__init__.py b/python_modules/dagster-externals/dagster_externals_tests/__init__.py similarity index 100% rename from python_modules/dagster-external/dagster_external_tests/__init__.py rename to python_modules/dagster-externals/dagster_externals_tests/__init__.py diff --git a/python_modules/dagster-external/dagster_external_tests/test_context.py b/python_modules/dagster-externals/dagster_externals_tests/test_context.py similarity index 96% rename from python_modules/dagster-external/dagster_external_tests/test_context.py rename to python_modules/dagster-externals/dagster_externals_tests/test_context.py index 5d7ae211354a8..872efe68072c7 100644 --- a/python_modules/dagster-external/dagster_external_tests/test_context.py +++ b/python_modules/dagster-externals/dagster_externals_tests/test_context.py @@ -1,14 +1,14 @@ from unittest.mock import MagicMock import pytest -from dagster_external.context import ExternalExecutionContext -from dagster_external.protocol import ( +from dagster_externals._context import ExternalExecutionContext +from dagster_externals._protocol import ( ExternalDataProvenance, ExternalExecutionContextData, ExternalPartitionKeyRange, ExternalTimeWindow, ) -from dagster_external.util import DagsterExternalError +from dagster_externals._util import DagsterExternalError TEST_EXTERNAL_EXECUTION_CONTEXT_DEFAULTS = ExternalExecutionContextData( asset_keys=None, diff --git a/python_modules/dagster-external/dagster_external_tests/test_external_execution.py b/python_modules/dagster-externals/dagster_externals_tests/test_external_execution.py similarity index 93% rename from python_modules/dagster-external/dagster_external_tests/test_external_execution.py rename to python_modules/dagster-externals/dagster_externals_tests/test_external_execution.py index fd70a6d5f7e6b..8a00f86db4299 100644 --- a/python_modules/dagster-external/dagster_external_tests/test_external_execution.py +++ b/python_modules/dagster-externals/dagster_externals_tests/test_external_execution.py @@ -47,9 +47,9 @@ def test_external_subprocess_asset(input_file_spec: str, output_file_spec: str, output_path = None if output_file_spec == "auto" else str(tmpdir.join("output")) def script_fn(): - from dagster_external import ExternalExecutionContext, init_dagster_external + from dagster_externals import ExternalExecutionContext, init_dagster_externals - init_dagster_external() + init_dagster_externals() context = ExternalExecutionContext.get() context.log("hello world") context.report_asset_metadata("foo", "bar", context.get_extra("bar")) @@ -99,9 +99,9 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource): def test_external_execution_asset_invocation(): def script_fn(): - from dagster_external import init_dagster_external + from dagster_externals import init_dagster_externals - context = init_dagster_external() + context = init_dagster_externals() context.log("hello world") @asset @@ -146,15 +146,15 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource): def test_external_execution_no_orchestration(): def script_fn(): - from dagster_external import ( + from dagster_externals import ( ExternalExecutionContext, - init_dagster_external, + init_dagster_externals, is_dagster_orchestration_active, ) assert not is_dagster_orchestration_active() - init_dagster_external() + init_dagster_externals() context = ExternalExecutionContext.get() context.log("hello world") context.report_asset_metadata("foo", "bar", context.get_extra("bar")) @@ -167,6 +167,6 @@ def script_fn(): ).communicate() assert re.search( r"This process was not launched by a Dagster orchestration process. All calls to the" - r" `dagster-external` context are no-ops.", + r" `dagster-externals` context are no-ops.", stderr.decode(), ) diff --git a/python_modules/dagster-externals/dagster_externals_tests/test_version.py b/python_modules/dagster-externals/dagster_externals_tests/test_version.py new file mode 100644 index 0000000000000..a339f1316872b --- /dev/null +++ b/python_modules/dagster-externals/dagster_externals_tests/test_version.py @@ -0,0 +1,5 @@ +from dagster_externals.version import __version__ + + +def test_version(): + assert __version__ diff --git a/python_modules/dagster-external/setup.cfg b/python_modules/dagster-externals/setup.cfg similarity index 100% rename from python_modules/dagster-external/setup.cfg rename to python_modules/dagster-externals/setup.cfg diff --git a/python_modules/dagster-external/setup.py b/python_modules/dagster-externals/setup.py similarity index 88% rename from python_modules/dagster-external/setup.py rename to python_modules/dagster-externals/setup.py index 0cb22f3d3f5c3..793c97f7272ba 100644 --- a/python_modules/dagster-external/setup.py +++ b/python_modules/dagster-externals/setup.py @@ -6,7 +6,7 @@ def get_version() -> str: version: Dict[str, str] = {} - with open(Path(__file__).parent / "dagster_external/version.py", encoding="utf8") as fp: + with open(Path(__file__).parent / "dagster_externals/version.py", encoding="utf8") as fp: exec(fp.read(), version) return version["__version__"] @@ -16,13 +16,13 @@ def get_version() -> str: # dont pin dev installs to avoid pip dep resolver issues pin = "" if ver == "1!0+dev" else f"=={ver}" setup( - name="dagster_external", + name="dagster-externals", version=get_version(), author="Elementl", author_email="hello@elementl.com", license="Apache-2.0", description="Toolkit for Dagster integrations with transform logic outside of Dagster", - url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-external", + url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-externals", classifiers=[ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/python_modules/dagster-external/tox.ini b/python_modules/dagster-externals/tox.ini similarity index 86% rename from python_modules/dagster-external/tox.ini rename to python_modules/dagster-externals/tox.ini index 87201b7664da0..a48293e39e8d5 100644 --- a/python_modules/dagster-external/tox.ini +++ b/python_modules/dagster-externals/tox.ini @@ -15,4 +15,4 @@ allowlist_externals = /bin/bash commands = !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster' - pytest -c ../../pyproject.toml -vv ./dagster_external_tests + pytest -c ../../pyproject.toml -vv ./dagster_externals_tests diff --git a/python_modules/dagster-test/Dockerfile b/python_modules/dagster-test/Dockerfile index 9c739cbc82ef4..d7ae670772bb6 100644 --- a/python_modules/dagster-test/Dockerfile +++ b/python_modules/dagster-test/Dockerfile @@ -32,7 +32,7 @@ RUN pip install \ -e modules/dagster-docker \ -e modules/dagster-airflow \ -e modules/dagstermill \ - -e modules/dagster-external \ + -e modules/dagster-externals \ -e . \ pyparsing\<3.0.0 diff --git a/python_modules/dagster-test/dagster_test/test_project/build.sh b/python_modules/dagster-test/dagster_test/test_project/build.sh index db618f062ce08..8bc2a311d0aa5 100755 --- a/python_modules/dagster-test/dagster_test/test_project/build.sh +++ b/python_modules/dagster-test/dagster_test/test_project/build.sh @@ -48,7 +48,7 @@ alias copy_py="rsync -av \ copy_py $ROOT/python_modules/dagster \ $ROOT/python_modules/dagster-webserver \ $ROOT/python_modules/dagster-graphql \ - $ROOT/python_modules/dagster-external \ + $ROOT/python_modules/dagster-externals \ $ROOT/python_modules/libraries/dagster-airflow \ $ROOT/python_modules/libraries/dagster-aws \ $ROOT/python_modules/libraries/dagster-celery \ diff --git a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py index c9743f2dcbdec..298537bf928e3 100644 --- a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py +++ b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py @@ -1,8 +1,8 @@ -from dagster_external import init_dagster_external +from dagster_externals import init_dagster_externals from .util import compute_data_version, load_asset_value, store_asset_value -context = init_dagster_external() +context = init_dagster_externals() storage_root = context.get_extra("storage_root") number_y = load_asset_value("number_y", storage_root) number_x = load_asset_value("number_x", storage_root) diff --git a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py index 3f0719aec37d3..8819de4548fd4 100644 --- a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py +++ b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py @@ -1,8 +1,8 @@ -from dagster_external import init_dagster_external +from dagster_externals import init_dagster_externals from .util import compute_data_version, store_asset_value -context = init_dagster_external() +context = init_dagster_externals() storage_root = context.get_extra("storage_root") multiplier = context.get_extra("multiplier") diff --git a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py index 4d8ff6c861cc5..73704b72dd511 100644 --- a/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py +++ b/python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py @@ -1,10 +1,10 @@ import os -from dagster_external import init_dagster_external +from dagster_externals import init_dagster_externals from .util import compute_data_version, store_asset_value -context = init_dagster_external() +context = init_dagster_externals() storage_root = context.get_extra("storage_root") value = int(os.environ["NUMBER_Y"]) diff --git a/python_modules/dagster/dagster/_core/external_execution/context.py b/python_modules/dagster/dagster/_core/external_execution/context.py index a40a4f25061e7..1736c9fe079be 100644 --- a/python_modules/dagster/dagster/_core/external_execution/context.py +++ b/python_modules/dagster/dagster/_core/external_execution/context.py @@ -1,6 +1,6 @@ from typing import Optional -from dagster_external.protocol import ( +from dagster_externals import ( ExternalDataProvenance, ExternalExecutionContextData, ExternalExecutionExtras, diff --git a/python_modules/dagster/dagster/_core/external_execution/resource.py b/python_modules/dagster/dagster/_core/external_execution/resource.py index 7f4342f5d1ffe..bec542090b140 100644 --- a/python_modules/dagster/dagster/_core/external_execution/resource.py +++ b/python_modules/dagster/dagster/_core/external_execution/resource.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod from typing import Optional -from dagster_external.protocol import ExternalExecutionExtras +from dagster_externals import ExternalExecutionExtras from pydantic import Field from dagster._config.pythonic_config import ConfigurableResource diff --git a/python_modules/dagster/dagster/_core/external_execution/subprocess.py b/python_modules/dagster/dagster/_core/external_execution/subprocess.py index 912e80113e72d..fa87bab042461 100644 --- a/python_modules/dagster/dagster/_core/external_execution/subprocess.py +++ b/python_modules/dagster/dagster/_core/external_execution/subprocess.py @@ -5,8 +5,8 @@ from threading import Event from typing import ContextManager, Iterator, Mapping, Optional, Sequence, Union -from dagster_external.protocol import ( - DAGSTER_EXTERNAL_ENV_KEYS, +from dagster_externals import ( + DAGSTER_EXTERNALS_ENV_KEYS, ExternalExecutionExtras, ) from pydantic import Field @@ -74,7 +74,7 @@ def _input_context_manager( @contextmanager def _file_input(self, tempdir: str) -> Iterator[SubprocessTaskIOParams]: path = self._prepare_io_path(self._input_path, "input", tempdir) - env = {DAGSTER_EXTERNAL_ENV_KEYS["input"]: path} + env = {DAGSTER_EXTERNALS_ENV_KEYS["input"]: path} try: self._write_input(path) yield SubprocessTaskIOParams(env=env) @@ -90,7 +90,7 @@ def _output_context_manager( @contextmanager def _file_output(self, tempdir: str) -> Iterator[SubprocessTaskIOParams]: path = self._prepare_io_path(self._output_path, "output", tempdir) - env = {DAGSTER_EXTERNAL_ENV_KEYS["output"]: path} + env = {DAGSTER_EXTERNALS_ENV_KEYS["output"]: path} is_task_complete = Event() thread = None try: diff --git a/python_modules/dagster/dagster/_core/external_execution/task.py b/python_modules/dagster/dagster/_core/external_execution/task.py index c0bc75fdc1042..974c1c2038d0c 100644 --- a/python_modules/dagster/dagster/_core/external_execution/task.py +++ b/python_modules/dagster/dagster/_core/external_execution/task.py @@ -17,10 +17,10 @@ Union, ) -from dagster_external.protocol import ( - DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME, - DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME, - DAGSTER_EXTERNAL_ENV_KEYS, +from dagster_externals import ( + DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME, + DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME, + DAGSTER_EXTERNALS_ENV_KEYS, ExternalExecutionExtras, ) from typing_extensions import Literal, TypeAlias, TypeVar @@ -74,7 +74,7 @@ def run(self, params: T_TaskParams) -> None: def get_base_env(self) -> Mapping[str, str]: return { **os.environ, - DAGSTER_EXTERNAL_ENV_KEYS["is_orchestration_active"]: "1", + DAGSTER_EXTERNALS_ENV_KEYS["is_orchestration_active"]: "1", } # ######################## @@ -167,9 +167,9 @@ def _prepare_io_path( ) -> str: if path is None: if target == "input": - filename = DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME + filename = DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME else: # output - filename = DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME + filename = DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME assert tempdir is not None, "Must define tempdir when path is None" return os.path.join(tempdir, filename) else: diff --git a/python_modules/libraries/dagster-docker/dagster_docker/external_resource.py b/python_modules/libraries/dagster-docker/dagster_docker/external_resource.py index 40f02adaa73fb..bd2437fb8fc99 100644 --- a/python_modules/libraries/dagster-docker/dagster_docker/external_resource.py +++ b/python_modules/libraries/dagster-docker/dagster_docker/external_resource.py @@ -15,11 +15,11 @@ ExternalTaskParams, SocketAddress, ) -from dagster_external.protocol import ( - DAGSTER_EXTERNAL_ENV_KEYS, +from dagster_externals import ( + DAGSTER_EXTERNALS_ENV_KEYS, + DagsterExternalError, ExternalExecutionExtras, ) -from dagster_external.util import DagsterExternalError @dataclass @@ -95,7 +95,7 @@ def _input_context_manager( @contextmanager def _file_input(self, tempdir: str) -> Iterator[DockerTaskIOParams]: path = self._prepare_io_path(self._input_path, "input", tempdir) - env = {DAGSTER_EXTERNAL_ENV_KEYS["input"]: path} + env = {DAGSTER_EXTERNALS_ENV_KEYS["input"]: path} try: self._write_input(path) path_dir = os.path.dirname(path) @@ -113,7 +113,7 @@ def _output_context_manager( @contextmanager def _file_output(self, tempdir: str) -> Iterator[DockerTaskIOParams]: path = self._prepare_io_path(self._output_path, "output", tempdir) - env = {DAGSTER_EXTERNAL_ENV_KEYS["output"]: path} + env = {DAGSTER_EXTERNALS_ENV_KEYS["output"]: path} output_file_dir = os.path.dirname(path) volume_mounts = {output_file_dir: {"bind": output_file_dir, "mode": "rw"}} is_task_complete = Event() diff --git a/python_modules/libraries/dagster-docker/tox.ini b/python_modules/libraries/dagster-docker/tox.ini index cdc9f02b4a7c1..3308db1b504c0 100644 --- a/python_modules/libraries/dagster-docker/tox.ini +++ b/python_modules/libraries/dagster-docker/tox.ini @@ -6,7 +6,7 @@ download = True passenv = HOME CI_* COVERALLS_REPO_TOKEN BUILDKITE* AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID DAGSTER_DOCKER_* DOCKER_* GOOGLE_* POSTGRES_TEST_DB_HOST deps = -e ../../dagster[test] - -e ../../dagster-external + -e ../../dagster-externals -e ../../dagster-graphql -e ../../dagster-test -e ../dagster-aws @@ -17,7 +17,7 @@ deps = -e ../dagster-k8s -e ../dagster-celery-k8s -e ../dagster-postgres - -e ../../dagster-external + -e ../../dagster-externals -e . allowlist_externals = /bin/bash diff --git a/scripts/install_dev_python_modules.py b/scripts/install_dev_python_modules.py index 2186cfca5c523..231eef33adb82 100644 --- a/scripts/install_dev_python_modules.py +++ b/scripts/install_dev_python_modules.py @@ -41,7 +41,7 @@ def main( # Supported on all Python versions. install_targets += [ "-e python_modules/dagster[black,pyright,ruff,test]", - "-e python_modules/dagster-external", + "-e python_modules/dagster-externals", "-e python_modules/dagster-graphql", "-e python_modules/dagster-test", "-e python_modules/dagster-webserver",