Skip to content

Commit

Permalink
minor ext fixes (#16684)
Browse files Browse the repository at this point in the history
## Summary & Motivation

Grab bag of non-user-facing fixes for consistency in variable names etc.

## How I Tested These Changes

Existing test suite.
  • Loading branch information
smackesey authored and prha committed Sep 22, 2023
1 parent c464e3a commit 98a476b
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 14 deletions.
2 changes: 1 addition & 1 deletion python_modules/dagster-ext/MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
include README.md
include LICENSE
include dagster_external/py.typed
include dagster_ext/py.typed
6 changes: 3 additions & 3 deletions python_modules/dagster-ext/dagster_ext/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ class ExtMessage(TypedDict):
params: Optional[Mapping[str, Any]]


# ##### EXTERNAL EXECUTION CONTEXT
# ##### EXT CONTEXT


class ExtContextData(TypedDict):
Expand Down Expand Up @@ -624,11 +624,11 @@ def __init__(
message_channel: ExtMessageWriterChannel,
) -> None:
self._data = data
self.message_channel = message_channel
self._message_channel = message_channel

def _write_message(self, method: str, params: Optional[Mapping[str, Any]] = None) -> None:
message = ExtMessage(method=method, params=params)
self.message_channel.write_message(message)
self._message_channel.write_message(message)

# ########################
# ##### PUBLIC API
Expand Down
11 changes: 4 additions & 7 deletions python_modules/dagster/dagster/_core/ext/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,6 @@
_CONTEXT_INJECTOR_FILENAME = "context"
_MESSAGE_READER_FILENAME = "messages"

_CONTEXT_INJECTOR_FILENAME = "context"
_MESSAGE_READER_FILENAME = "messages"


class ExtFileContextInjector(ExtContextInjector):
def __init__(self, path: str):
Expand Down Expand Up @@ -112,7 +109,7 @@ def read_messages(
) -> Iterator[ExtParams]:
with tempfile.TemporaryDirectory() as tempdir:
with ExtFileMessageReader(
os.path.join(tempdir, _CONTEXT_INJECTOR_FILENAME)
os.path.join(tempdir, _MESSAGE_READER_FILENAME)
).read_messages(handler) as params:
yield params

Expand Down Expand Up @@ -199,15 +196,15 @@ def ext_protocol(
that need to be provided to the external process.
"""
context_data = build_external_execution_context_data(context, extras)
msg_handler = ExtMessageHandler(context)
message_handler = ExtMessageHandler(context)
with context_injector.inject_context(
context_data,
) as ci_params, message_reader.read_messages(
msg_handler,
message_handler,
) as mr_params:
yield ExtOrchestrationContext(
context_data=context_data,
message_handler=msg_handler,
message_handler=message_handler,
context_injector_params=ci_params,
message_reader_params=mr_params,
)
2 changes: 1 addition & 1 deletion python_modules/libraries/dagster-aws/dagster_aws/ext.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@


class ExtS3MessageReader(ExtBlobStoreMessageReader):
def __init__(self, *, interval: int = 10, bucket: str, client: boto3.client):
def __init__(self, *, interval: float = 10, bucket: str, client: boto3.client):
super().__init__(interval=interval)
self.bucket = check.str_param(bucket, "bucket")
self.key_prefix = "".join(random.choices(string.ascii_letters, k=30))
Expand Down
4 changes: 2 additions & 2 deletions python_modules/libraries/dagster-k8s/dagster_k8s/ext.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,15 +158,15 @@ def run(
extras=extras,
context_injector=self.context_injector,
message_reader=self.message_reader,
) as ext_process:
) as ext_context:
namespace = namespace or "default"
pod_name = get_pod_name(context.run_id, context.op.name)
pod_body = build_pod_body(
pod_name=pod_name,
image=image,
command=command,
env_vars={
**ext_process.get_external_process_env_vars(),
**ext_context.get_external_process_env_vars(),
**(self.env or {}),
**(env or {}),
},
Expand Down

0 comments on commit 98a476b

Please sign in to comment.