[externals] Rename dagster-external -> dagster-externals

examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py

@@ -1,6 +1,6 @@
 import sys
 
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_external
 
 
 class SomeSqlClient:

examples/experimental/assets_yaml_dsl/tox.ini

@@ -6,7 +6,7 @@ download = True
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
 deps =
   -e ../../../python_modules/dagster[test]
-  -e ../../../python_modules/dagster-external
+  -e ../../../python_modules/dagster-externals
   -e ../../../python_modules/dagster-webserver
   -e ../../../python_modules/dagster-graphql
   -e .

pyright/master/requirements-pinned.txt

@@ -90,7 +90,7 @@ cryptography==41.0.2
 cycler==0.11.0
 -e python_modules/dagit
 -e python_modules/dagster
--e python_modules/dagster-external
+-e python_modules/dagster-externals
 -e python_modules/libraries/dagster-airbyte
 -e python_modules/libraries/dagster-airflow
 -e python_modules/libraries/dagster-aws

pyright/master/requirements.txt

@@ -31,7 +31,7 @@
 -e python_modules/dagit
 -e python_modules/dagster-graphql
 -e python_modules/dagster-test
--e python_modules/dagster-external
+-e python_modules/dagster-externals
 -e python_modules/dagster-webserver
 -e python_modules/libraries/dagster-airbyte/
 -e python_modules/libraries/dagster-airflow[test_airflow_2]

python_modules/dagster-externals/README.md

@@ -0,0 +1,4 @@
+# dagster-externals
+
+The docs for `dagster-externals ` can be found
+[here](https://docs.dagster.io/_apidocs/libraries/dagster-externals).

python_modules/dagster-externals/dagster_externals/__init__.py

@@ -0,0 +1,15 @@
+from dagster_externals._context import (
+    ExternalExecutionContext as ExternalExecutionContext,
+    init_dagster_external as init_dagster_external,
+    is_dagster_orchestration_active as is_dagster_orchestration_active,
+)
+from dagster_externals._protocol import (
+    DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME,
+    DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME,
+    DAGSTER_EXTERNALS_ENV_KEYS as DAGSTER_EXTERNALS_ENV_KEYS,
+    ExternalDataProvenance as ExternalDataProvenance,
+    ExternalExecutionContextData as ExternalExecutionContextData,
+    ExternalPartitionKeyRange as ExternalPartitionKeyRange,
+    ExternalTimeWindow as ExternalTimeWindow,
+    Notification as Notification,
+)

python_modules/dagster-external/dagster_external/context.py → python_modules/dagster-externals/dagster_externals/_context.py

@@ -6,10 +6,9 @@
 
 from typing_extensions import Self
 
-from dagster_external.params import get_external_execution_params
-
-from .protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from ._params import get_external_execution_params
+from ._protocol import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
     ExternalDataProvenance,
     ExternalExecutionContextData,
     ExternalPartitionKeyRange,
@@ -28,7 +27,7 @@
 
 
 def is_dagster_orchestration_active() -> bool:
-    return bool(os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["is_orchestration_active"]))
+    return bool(os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["is_orchestration_active"]))
 
 
 def init_dagster_external() -> "ExternalExecutionContext":
@@ -46,7 +45,7 @@ def init_dagster_external() -> "ExternalExecutionContext":
 
         warnings.warn(
             "This process was not launched by a Dagster orchestration process. All calls to the"
-            " `dagster-external` context are no-ops."
+            " `dagster-externals` context are no-ops."
         )
         context = MagicMock()
     ExternalExecutionContext.set(context)

python_modules/dagster-external/dagster_external/params.py → python_modules/dagster-externals/dagster_externals/_params.py

@@ -1,14 +1,14 @@
 import os
 from dataclasses import dataclass
 
-from dagster_external.protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from ._protocol import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
 )
 
 
 def get_external_execution_params() -> "ExternalExecutionParams":
-    input_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["input"])
-    output_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["output"])
+    input_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["input"])
+    output_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["output"])
     assert input_path, "input_path must be set"
     assert output_path, "output_path must be set"
     return ExternalExecutionParams(

python_modules/dagster-external/dagster_external/protocol.py → python_modules/dagster-externals/dagster_externals/_protocol.py

@@ -6,13 +6,13 @@
 
 # ##### PARAMETERS
 
-DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input"
-DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output"
+DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input"
+DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output"
 
-DAGSTER_EXTERNAL_ENV_KEYS: Final = {
-    "is_orchestration_active": "DAGSTER_EXTERNAL_IS_ORCHESTRATION_ACTIVE",
-    "input": "DAGSTER_EXTERNAL_INPUT",
-    "output": "DAGSTER_EXTERNAL_OUTPUT",
+DAGSTER_EXTERNALS_ENV_KEYS: Final = {
+    "is_orchestration_active": "DAGSTER_EXTERNALS_IS_ORCHESTRATION_ACTIVE",
+    "input": "DAGSTER_EXTERNALS_INPUT",
+    "output": "DAGSTER_EXTERNALS_OUTPUT",
 }
 
 # ##### NOTIFICATION

python_modules/dagster-external/dagster_external/util.py → python_modules/dagster-externals/dagster_externals/_util.py

@@ -1,7 +1,7 @@
 import json
 from typing import Any, Optional, Sequence, TypeVar
 
-from dagster_external.protocol import ExternalExecutionContextData, ExternalExecutionExtras
+from ._protocol import ExternalExecutionContextData, ExternalExecutionExtras
 
 T = TypeVar("T")
 

python_modules/dagster-external/dagster_external_tests/test_external_execution.py → python_modules/dagster-externals/dagster_externals_tests/test_external_execution.py

@@ -47,7 +47,7 @@ def test_external_subprocess_asset(input_file_spec: str, output_file_spec: str,
     output_path = None if output_file_spec == "auto" else str(tmpdir.join("output"))
 
     def script_fn():
-        from dagster_external import ExternalExecutionContext, init_dagster_external
+        from dagster_externals import ExternalExecutionContext, init_dagster_external
 
         init_dagster_external()
         context = ExternalExecutionContext.get()
@@ -99,7 +99,7 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource):
 
 def test_external_execution_asset_invocation():
     def script_fn():
-        from dagster_external import init_dagster_external
+        from dagster_externals import init_dagster_external
 
         context = init_dagster_external()
         context.log("hello world")
@@ -146,7 +146,7 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource):
 
 def test_external_execution_no_orchestration():
     def script_fn():
-        from dagster_external import (
+        from dagster_externals import (
             ExternalExecutionContext,
             init_dagster_external,
             is_dagster_orchestration_active,
@@ -167,6 +167,6 @@ def script_fn():
         ).communicate()
         assert re.search(
             r"This process was not launched by a Dagster orchestration process. All calls to the"
-            r" `dagster-external` context are no-ops.",
+            r" `dagster-externals` context are no-ops.",
             stderr.decode(),
         )

python_modules/dagster-externals/dagster_externals_tests/test_version.py

@@ -0,0 +1,5 @@
+from dagster_externals.version import __version__
+
+
+def test_version():
+    assert __version__

python_modules/dagster-external/setup.py → python_modules/dagster-externals/setup.py

@@ -6,7 +6,7 @@
 
 def get_version() -> str:
     version: Dict[str, str] = {}
-    with open(Path(__file__).parent / "dagster_external/version.py", encoding="utf8") as fp:
+    with open(Path(__file__).parent / "dagster_externals/version.py", encoding="utf8") as fp:
         exec(fp.read(), version)
 
     return version["__version__"]
@@ -16,13 +16,13 @@ def get_version() -> str:
 # dont pin dev installs to avoid pip dep resolver issues
 pin = "" if ver == "1!0+dev" else f"=={ver}"
 setup(
-    name="dagster_external",
+    name="dagster-externals",
     version=get_version(),
     author="Elementl",
     author_email="[email protected]",
     license="Apache-2.0",
     description="Toolkit for Dagster integrations with transform logic outside of Dagster",
-    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-external",
+    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-externals",
     classifiers=[
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",

python_modules/dagster-external/tox.ini → python_modules/dagster-externals/tox.ini

@@ -15,4 +15,4 @@ allowlist_externals =
   /bin/bash
 commands =
   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
-  pytest -c ../../pyproject.toml -vv ./dagster_external_tests
+  pytest -c ../../pyproject.toml -vv ./dagster_externals_tests

python_modules/dagster-test/Dockerfile

@@ -32,7 +32,7 @@ RUN pip install \
     -e modules/dagster-docker \
     -e modules/dagster-airflow \
     -e modules/dagstermill \
-    -e modules/dagster-external \
+    -e modules/dagster-externals \
     -e . \
     pyparsing\<3.0.0
 

python_modules/dagster-test/dagster_test/test_project/build.sh

@@ -48,7 +48,7 @@ alias copy_py="rsync -av \
 copy_py $ROOT/python_modules/dagster \
         $ROOT/python_modules/dagster-webserver \
         $ROOT/python_modules/dagster-graphql \
-        $ROOT/python_modules/dagster-external \
+        $ROOT/python_modules/dagster-externals \
         $ROOT/python_modules/libraries/dagster-airflow \
         $ROOT/python_modules/libraries/dagster-aws \
         $ROOT/python_modules/libraries/dagster-celery \

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py

@@ -1,4 +1,4 @@
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_external
 
 from .util import compute_data_version, load_asset_value, store_asset_value
 

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py

@@ -1,4 +1,4 @@
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_external
 
 from .util import compute_data_version, store_asset_value
 

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py

@@ -1,6 +1,6 @@
 import os
 
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_external
 
 from .util import compute_data_version, store_asset_value
 

python_modules/dagster/dagster/_core/external_execution/context.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from dagster_external.protocol import (
+from dagster_externals import (
     ExternalDataProvenance,
     ExternalExecutionContextData,
     ExternalExecutionExtras,

python_modules/dagster/dagster/_core/external_execution/resource.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Optional
 
-from dagster_external.protocol import ExternalExecutionExtras
+from dagster_externals import ExternalExecutionExtras
 from pydantic import Field
 
 from dagster._config.pythonic_config import ConfigurableResource

python_modules/dagster/dagster/_core/external_execution/subprocess.py

@@ -5,8 +5,8 @@
 from threading import Event
 from typing import ContextManager, Iterator, Mapping, Optional, Sequence, Union
 
-from dagster_external.protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from dagster_externals import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
     ExternalExecutionExtras,
 )
 from pydantic import Field
@@ -74,7 +74,7 @@ def _input_context_manager(
     @contextmanager
     def _file_input(self, tempdir: str) -> Iterator[SubprocessTaskIOParams]:
         path = self._prepare_io_path(self._input_path, "input", tempdir)
-        env = {DAGSTER_EXTERNAL_ENV_KEYS["input"]: path}
+        env = {DAGSTER_EXTERNALS_ENV_KEYS["input"]: path}
         try:
             self._write_input(path)
             yield SubprocessTaskIOParams(env=env)
@@ -90,7 +90,7 @@ def _output_context_manager(
     @contextmanager
     def _file_output(self, tempdir: str) -> Iterator[SubprocessTaskIOParams]:
         path = self._prepare_io_path(self._output_path, "output", tempdir)
-        env = {DAGSTER_EXTERNAL_ENV_KEYS["output"]: path}
+        env = {DAGSTER_EXTERNALS_ENV_KEYS["output"]: path}
         is_task_complete = Event()
         thread = None
         try:

python_modules/dagster/dagster/_core/external_execution/task.py

@@ -17,10 +17,10 @@
     Union,
 )
 
-from dagster_external.protocol import (
-    DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME,
-    DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME,
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from dagster_externals import (
+    DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME,
+    DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME,
+    DAGSTER_EXTERNALS_ENV_KEYS,
     ExternalExecutionExtras,
 )
 from typing_extensions import Literal, TypeAlias, TypeVar
@@ -74,7 +74,7 @@ def run(self, params: T_TaskParams) -> None:
     def get_base_env(self) -> Mapping[str, str]:
         return {
             **os.environ,
-            DAGSTER_EXTERNAL_ENV_KEYS["is_orchestration_active"]: "1",
+            DAGSTER_EXTERNALS_ENV_KEYS["is_orchestration_active"]: "1",
         }
 
     # ########################
@@ -167,9 +167,9 @@ def _prepare_io_path(
     ) -> str:
         if path is None:
             if target == "input":
-                filename = DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME
+                filename = DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME
             else:  # output
-                filename = DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME
+                filename = DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME
             assert tempdir is not None, "Must define tempdir when path is None"
             return os.path.join(tempdir, filename)
         else:

python_modules/libraries/dagster-docker/dagster_docker/external_resource.py

@@ -15,11 +15,11 @@
     ExternalTaskParams,
     SocketAddress,
 )
-from dagster_external.protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from dagster_externals import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
     ExternalExecutionExtras,
 )
-from dagster_external.util import DagsterExternalError
+from dagster_externals.util import DagsterExternalError
 
 
 @dataclass
@@ -95,7 +95,7 @@ def _input_context_manager(
     @contextmanager
     def _file_input(self, tempdir: str) -> Iterator[DockerTaskIOParams]:
         path = self._prepare_io_path(self._input_path, "input", tempdir)
-        env = {DAGSTER_EXTERNAL_ENV_KEYS["input"]: path}
+        env = {DAGSTER_EXTERNALS_ENV_KEYS["input"]: path}
         try:
             self._write_input(path)
             path_dir = os.path.dirname(path)
@@ -113,7 +113,7 @@ def _output_context_manager(
     @contextmanager
     def _file_output(self, tempdir: str) -> Iterator[DockerTaskIOParams]:
         path = self._prepare_io_path(self._output_path, "output", tempdir)
-        env = {DAGSTER_EXTERNAL_ENV_KEYS["output"]: path}
+        env = {DAGSTER_EXTERNALS_ENV_KEYS["output"]: path}
         output_file_dir = os.path.dirname(path)
         volume_mounts = {output_file_dir: {"bind": output_file_dir, "mode": "rw"}}
         is_task_complete = Event()

python_modules/libraries/dagster-docker/tox.ini

@@ -6,7 +6,7 @@ download = True
 passenv = HOME CI_* COVERALLS_REPO_TOKEN BUILDKITE* AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID DAGSTER_DOCKER_* DOCKER_* GOOGLE_* POSTGRES_TEST_DB_HOST
 deps =
   -e ../../dagster[test]
-  -e ../../dagster-external
+  -e ../../dagster-externals
   -e ../../dagster-graphql
   -e ../../dagster-test
   -e ../dagster-aws
@@ -17,7 +17,7 @@ deps =
   -e ../dagster-k8s
   -e ../dagster-celery-k8s
   -e ../dagster-postgres
-  -e ../../dagster-external
+  -e ../../dagster-externals
   -e .
 allowlist_externals =
   /bin/bash