[externals] Rename dagster-external -> dagster-externals

[INTERNAL_BRANCH=sean/externals-rename-external]

examples/experimental/assets_yaml_dsl/assets_yaml_dsl/sql_script.py

@@ -1,6 +1,6 @@
 import sys
 
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_externals
 
 
 class SomeSqlClient:
@@ -11,7 +11,7 @@ def query(self, query_str: str) -> None:
 if __name__ == "__main__":
     sql = sys.argv[1]
 
-    context = init_dagster_external()
+    context = init_dagster_externals()
 
     client = SomeSqlClient()
     client.query(sql)

examples/experimental/assets_yaml_dsl/setup.py

@@ -3,7 +3,7 @@
 setup(
     name="assets_yaml_dsl",
     packages=find_packages(exclude=["assets_yaml_dsl_tests"]),
-    install_requires=["dagster", "pandas", "dagster_external"],
+    install_requires=["dagster", "pandas", "dagster_externals"],
     license="Apache-2.0",
     description="Dagster example of yaml dsl for building asset graphs",
     url="https://github.com/dagster-io/dagster/tree/master/examples/assets_yaml_dsl",

examples/experimental/assets_yaml_dsl/tox.ini

@@ -6,7 +6,7 @@ download = True
 passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
 deps =
   -e ../../../python_modules/dagster[test]
-  -e ../../../python_modules/dagster-external
+  -e ../../../python_modules/dagster-externals
   -e ../../../python_modules/dagster-webserver
   -e ../../../python_modules/dagster-graphql
   -e .

pyright/master/requirements-pinned.txt

@@ -90,7 +90,7 @@ cryptography==41.0.2
 cycler==0.11.0
 -e python_modules/dagit
 -e python_modules/dagster
--e python_modules/dagster-external
+-e python_modules/dagster-externals
 -e python_modules/libraries/dagster-airbyte
 -e python_modules/libraries/dagster-airflow
 -e python_modules/libraries/dagster-aws

pyright/master/requirements.txt

@@ -31,7 +31,7 @@
 -e python_modules/dagit
 -e python_modules/dagster-graphql
 -e python_modules/dagster-test
--e python_modules/dagster-external
+-e python_modules/dagster-externals
 -e python_modules/dagster-webserver
 -e python_modules/libraries/dagster-airbyte/
 -e python_modules/libraries/dagster-airflow[test_airflow_2]

python_modules/dagster-externals/README.md

@@ -0,0 +1,4 @@
+# dagster-externals
+
+The docs for `dagster-externals ` can be found
+[here](https://docs.dagster.io/_apidocs/libraries/dagster-externals).

python_modules/dagster-externals/dagster_externals/__init__.py

@@ -0,0 +1,19 @@
+from dagster_externals._context import (
+    ExternalExecutionContext as ExternalExecutionContext,
+    init_dagster_externals as init_dagster_externals,
+    is_dagster_orchestration_active as is_dagster_orchestration_active,
+)
+from dagster_externals._protocol import (
+    DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME,
+    DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME as DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME,
+    DAGSTER_EXTERNALS_ENV_KEYS as DAGSTER_EXTERNALS_ENV_KEYS,
+    ExternalDataProvenance as ExternalDataProvenance,
+    ExternalExecutionContextData as ExternalExecutionContextData,
+    ExternalExecutionExtras as ExternalExecutionExtras,
+    ExternalPartitionKeyRange as ExternalPartitionKeyRange,
+    ExternalTimeWindow as ExternalTimeWindow,
+    Notification as Notification,
+)
+from dagster_externals._util import (
+    DagsterExternalError as DagsterExternalError,
+)

python_modules/dagster-external/dagster_external/context.py → python_modules/dagster-externals/dagster_externals/_context.py

@@ -6,17 +6,16 @@
 
 from typing_extensions import Self
 
-from dagster_external.params import get_external_execution_params
-
-from .protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from ._params import get_external_execution_params
+from ._protocol import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
     ExternalDataProvenance,
     ExternalExecutionContextData,
     ExternalPartitionKeyRange,
     ExternalTimeWindow,
     Notification,
 )
-from .util import (
+from ._util import (
     assert_defined_asset_property,
     assert_defined_extra,
     assert_defined_partition_property,
@@ -28,10 +27,10 @@
 
 
 def is_dagster_orchestration_active() -> bool:
-    return bool(os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["is_orchestration_active"]))
+    return bool(os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["is_orchestration_active"]))
 
 
-def init_dagster_external() -> "ExternalExecutionContext":
+def init_dagster_externals() -> "ExternalExecutionContext":
     if ExternalExecutionContext.is_initialized():
         return ExternalExecutionContext.get()
 
@@ -46,7 +45,7 @@ def init_dagster_external() -> "ExternalExecutionContext":
 
         warnings.warn(
             "This process was not launched by a Dagster orchestration process. All calls to the"
-            " `dagster-external` context are no-ops."
+            " `dagster-externals` context are no-ops."
         )
         context = MagicMock()
     ExternalExecutionContext.set(context)
@@ -82,7 +81,7 @@ def get(cls) -> Self:
         if cls._instance is None:
             raise Exception(
                 "ExternalExecutionContext has not been initialized. You must call"
-                " `init_dagster_external()`."
+                " `init_dagster_externals()`."
             )
         return cls._instance
 

python_modules/dagster-external/dagster_external/params.py → python_modules/dagster-externals/dagster_externals/_params.py

@@ -1,14 +1,14 @@
 import os
 from dataclasses import dataclass
 
-from dagster_external.protocol import (
-    DAGSTER_EXTERNAL_ENV_KEYS,
+from ._protocol import (
+    DAGSTER_EXTERNALS_ENV_KEYS,
 )
 
 
 def get_external_execution_params() -> "ExternalExecutionParams":
-    input_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["input"])
-    output_path = os.getenv(DAGSTER_EXTERNAL_ENV_KEYS["output"])
+    input_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["input"])
+    output_path = os.getenv(DAGSTER_EXTERNALS_ENV_KEYS["output"])
     assert input_path, "input_path must be set"
     assert output_path, "output_path must be set"
     return ExternalExecutionParams(

python_modules/dagster-external/dagster_external/protocol.py → python_modules/dagster-externals/dagster_externals/_protocol.py

@@ -6,13 +6,13 @@
 
 # ##### PARAMETERS
 
-DAGSTER_EXTERNAL_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input"
-DAGSTER_EXTERNAL_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output"
+DAGSTER_EXTERNALS_DEFAULT_INPUT_FILENAME: Final = "dagster_external_input"
+DAGSTER_EXTERNALS_DEFAULT_OUTPUT_FILENAME: Final = "dagster_external_output"
 
-DAGSTER_EXTERNAL_ENV_KEYS: Final = {
-    "is_orchestration_active": "DAGSTER_EXTERNAL_IS_ORCHESTRATION_ACTIVE",
-    "input": "DAGSTER_EXTERNAL_INPUT",
-    "output": "DAGSTER_EXTERNAL_OUTPUT",
+DAGSTER_EXTERNALS_ENV_KEYS: Final = {
+    "is_orchestration_active": "DAGSTER_EXTERNALS_IS_ORCHESTRATION_ACTIVE",
+    "input": "DAGSTER_EXTERNALS_INPUT",
+    "output": "DAGSTER_EXTERNALS_OUTPUT",
 }
 
 # ##### NOTIFICATION

python_modules/dagster-external/dagster_external/util.py → python_modules/dagster-externals/dagster_externals/_util.py

@@ -1,7 +1,7 @@
 import json
 from typing import Any, Optional, Sequence, TypeVar
 
-from dagster_external.protocol import ExternalExecutionContextData, ExternalExecutionExtras
+from ._protocol import ExternalExecutionContextData, ExternalExecutionExtras
 
 T = TypeVar("T")
 

python_modules/dagster-external/dagster_external_tests/test_context.py → python_modules/dagster-externals/dagster_externals_tests/test_context.py

@@ -1,14 +1,14 @@
 from unittest.mock import MagicMock
 
 import pytest
-from dagster_external.context import ExternalExecutionContext
-from dagster_external.protocol import (
+from dagster_externals._context import ExternalExecutionContext
+from dagster_externals._protocol import (
     ExternalDataProvenance,
     ExternalExecutionContextData,
     ExternalPartitionKeyRange,
     ExternalTimeWindow,
 )
-from dagster_external.util import DagsterExternalError
+from dagster_externals._util import DagsterExternalError
 
 TEST_EXTERNAL_EXECUTION_CONTEXT_DEFAULTS = ExternalExecutionContextData(
     asset_keys=None,

python_modules/dagster-external/dagster_external_tests/test_external_execution.py → python_modules/dagster-externals/dagster_externals_tests/test_external_execution.py

@@ -47,9 +47,9 @@ def test_external_subprocess_asset(input_file_spec: str, output_file_spec: str,
     output_path = None if output_file_spec == "auto" else str(tmpdir.join("output"))
 
     def script_fn():
-        from dagster_external import ExternalExecutionContext, init_dagster_external
+        from dagster_externals import ExternalExecutionContext, init_dagster_externals
 
-        init_dagster_external()
+        init_dagster_externals()
         context = ExternalExecutionContext.get()
         context.log("hello world")
         context.report_asset_metadata("foo", "bar", context.get_extra("bar"))
@@ -99,9 +99,9 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource):
 
 def test_external_execution_asset_invocation():
     def script_fn():
-        from dagster_external import init_dagster_external
+        from dagster_externals import init_dagster_externals
 
-        context = init_dagster_external()
+        context = init_dagster_externals()
         context.log("hello world")
 
     @asset
@@ -146,15 +146,15 @@ def foo(context: AssetExecutionContext, ext: SubprocessExecutionResource):
 
 def test_external_execution_no_orchestration():
     def script_fn():
-        from dagster_external import (
+        from dagster_externals import (
             ExternalExecutionContext,
-            init_dagster_external,
+            init_dagster_externals,
             is_dagster_orchestration_active,
         )
 
         assert not is_dagster_orchestration_active()
 
-        init_dagster_external()
+        init_dagster_externals()
         context = ExternalExecutionContext.get()
         context.log("hello world")
         context.report_asset_metadata("foo", "bar", context.get_extra("bar"))
@@ -167,6 +167,6 @@ def script_fn():
         ).communicate()
         assert re.search(
             r"This process was not launched by a Dagster orchestration process. All calls to the"
-            r" `dagster-external` context are no-ops.",
+            r" `dagster-externals` context are no-ops.",
             stderr.decode(),
         )

python_modules/dagster-externals/dagster_externals_tests/test_version.py

@@ -0,0 +1,5 @@
+from dagster_externals.version import __version__
+
+
+def test_version():
+    assert __version__

python_modules/dagster-external/setup.py → python_modules/dagster-externals/setup.py

@@ -6,7 +6,7 @@
 
 def get_version() -> str:
     version: Dict[str, str] = {}
-    with open(Path(__file__).parent / "dagster_external/version.py", encoding="utf8") as fp:
+    with open(Path(__file__).parent / "dagster_externals/version.py", encoding="utf8") as fp:
         exec(fp.read(), version)
 
     return version["__version__"]
@@ -16,13 +16,13 @@ def get_version() -> str:
 # dont pin dev installs to avoid pip dep resolver issues
 pin = "" if ver == "1!0+dev" else f"=={ver}"
 setup(
-    name="dagster_external",
+    name="dagster-externals",
     version=get_version(),
     author="Elementl",
     author_email="[email protected]",
     license="Apache-2.0",
     description="Toolkit for Dagster integrations with transform logic outside of Dagster",
-    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-external",
+    url="https://github.com/dagster-io/dagster/tree/master/python_modules/libraries/dagster-externals",
     classifiers=[
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",

python_modules/dagster-external/tox.ini → python_modules/dagster-externals/tox.ini

@@ -15,4 +15,4 @@ allowlist_externals =
   /bin/bash
 commands =
   !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster'
-  pytest -c ../../pyproject.toml -vv ./dagster_external_tests
+  pytest -c ../../pyproject.toml -vv ./dagster_externals_tests

python_modules/dagster-test/Dockerfile

@@ -32,7 +32,7 @@ RUN pip install \
     -e modules/dagster-docker \
     -e modules/dagster-airflow \
     -e modules/dagstermill \
-    -e modules/dagster-external \
+    -e modules/dagster-externals \
     -e . \
     pyparsing\<3.0.0
 

python_modules/dagster-test/dagster_test/test_project/build.sh

@@ -48,7 +48,7 @@ alias copy_py="rsync -av \
 copy_py $ROOT/python_modules/dagster \
         $ROOT/python_modules/dagster-webserver \
         $ROOT/python_modules/dagster-graphql \
-        $ROOT/python_modules/dagster-external \
+        $ROOT/python_modules/dagster-externals \
         $ROOT/python_modules/libraries/dagster-airflow \
         $ROOT/python_modules/libraries/dagster-aws \
         $ROOT/python_modules/libraries/dagster-celery \

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_sum.py

@@ -1,8 +1,8 @@
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_externals
 
 from .util import compute_data_version, load_asset_value, store_asset_value
 
-context = init_dagster_external()
+context = init_dagster_externals()
 storage_root = context.get_extra("storage_root")
 number_y = load_asset_value("number_y", storage_root)
 number_x = load_asset_value("number_x", storage_root)

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_x.py

@@ -1,8 +1,8 @@
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_externals
 
 from .util import compute_data_version, store_asset_value
 
-context = init_dagster_external()
+context = init_dagster_externals()
 storage_root = context.get_extra("storage_root")
 
 multiplier = context.get_extra("multiplier")

python_modules/dagster-test/dagster_test/toys/external_execution/numbers_example/number_y.py

@@ -1,10 +1,10 @@
 import os
 
-from dagster_external import init_dagster_external
+from dagster_externals import init_dagster_externals
 
 from .util import compute_data_version, store_asset_value
 
-context = init_dagster_external()
+context = init_dagster_externals()
 storage_root = context.get_extra("storage_root")
 
 value = int(os.environ["NUMBER_Y"])

python_modules/dagster/dagster/_core/external_execution/context.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from dagster_external.protocol import (
+from dagster_externals import (
     ExternalDataProvenance,
     ExternalExecutionContextData,
     ExternalExecutionExtras,

python_modules/dagster/dagster/_core/external_execution/resource.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Optional
 
-from dagster_external.protocol import ExternalExecutionExtras
+from dagster_externals import ExternalExecutionExtras
 from pydantic import Field
 
 from dagster._config.pythonic_config import ConfigurableResource