add PipesEMRClient

docs/sphinx/sections/api/apidocs/libraries/dagster-aws.rst

@@ -120,6 +120,8 @@ Clients
 
 .. autoclass:: dagster_aws.pipes.PipesGlueClient
 
+.. autoclass:: dagster_aws.pipes.PipesEMRClient
+
 Legacy
 --------
 

pyright/alt-1/requirements-pinned.txt

@@ -24,9 +24,9 @@ backports-tarfile==1.2.0
 beautifulsoup4==4.12.3
 bleach==6.1.0
 boto3==1.34.162
-boto3-stubs-lite==1.35.6
+boto3-stubs-lite==1.35.7
 botocore==1.34.162
-botocore-stubs==1.35.6
+botocore-stubs==1.35.7
 buildkite-test-collector==0.1.9
 cachetools==5.5.0
 caio==0.9.17
@@ -38,7 +38,7 @@ click==8.1.7
 colorama==0.4.6
 coloredlogs==14.0
 comm==0.2.2
-contourpy==1.2.1
+contourpy==1.3.0
 coverage==7.6.1
 croniter==3.0.3
 cryptography==43.0.0
@@ -65,8 +65,8 @@ daff==1.3.46
 -e python_modules/libraries/dagster-spark
 -e python_modules/dagster-webserver
 db-dtypes==1.3.0
-dbt-adapters==1.4.1
-dbt-common==1.7.0
+dbt-adapters==1.3.2
+dbt-common==1.3.0
 dbt-core==1.8.5
 dbt-duckdb==1.8.3
 dbt-extractor==0.5.1
@@ -75,7 +75,7 @@ dbt-snowflake==1.8.3
 debugpy==1.8.5
 decopatch==1.4.10
 decorator==5.1.1
-deepdiff==7.0.1
+deepdiff==8.0.0
 defusedxml==0.7.1
 deltalake==0.19.1
 dill==0.3.8
@@ -92,7 +92,7 @@ fqdn==1.5.1
 frozenlist==1.4.1
 fsspec==2024.3.1
 gcsfs==2024.3.1
-google-api-core==2.19.1
+google-api-core==2.19.2
 google-api-python-client==2.142.0
 google-auth==2.34.0
 google-auth-httplib2==0.2.0
@@ -102,7 +102,7 @@ google-cloud-core==2.4.1
 google-cloud-storage==2.18.2
 google-crc32c==1.5.0
 google-resumable-media==2.7.2
-googleapis-common-protos==1.64.0
+googleapis-common-protos==1.65.0
 gql==3.5.0
 graphene==3.3
 graphql-core==3.2.3
@@ -116,7 +116,7 @@ h11==0.14.0
 httpcore==1.0.5
 httplib2==0.22.0
 httptools==0.6.1
-httpx==0.27.0
+httpx==0.27.2
 humanfriendly==10.0
 hypothesis==6.111.2
 idna==3.8
@@ -171,6 +171,7 @@ multidict==6.0.5
 multimethod==1.10
 mypy==1.11.2
 mypy-boto3-ecs==1.35.2
+mypy-boto3-emr==1.35.0
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 nbclient==0.10.0
@@ -184,7 +185,7 @@ numpy==2.1.0
 oauth2client==4.1.3
 oauthlib==3.2.2
 objgraph==3.6.1
-ordered-set==4.1.0
+orderly-set==5.2.1
 orjson==3.10.7
 overrides==7.7.0
 packaging==24.1
@@ -226,6 +227,7 @@ pyproject-api==1.7.1
 pyright==1.1.370
 pyspark==3.5.2
 pytest==8.3.2
+pytest-asyncio==0.24.0
 pytest-cases==3.8.5
 pytest-cov==5.0.0
 pytest-mock==3.14.0
@@ -257,7 +259,7 @@ scipy==1.14.1
 seaborn==0.13.2
 secretstorage==3.3.3
 send2trash==1.8.3
-setuptools==73.0.1
+setuptools==74.0.0
 shellingham==1.5.4
 six==1.16.0
 slack-sdk==3.31.0
@@ -303,7 +305,7 @@ types-python-dateutil==2.9.0.20240821
 types-pytz==2024.1.0.20240417
 types-pyyaml==6.0.12.20240808
 types-requests==2.32.0.20240712
-types-s3transfer==0.10.1
+types-s3transfer==0.10.2
 types-setuptools==73.0.0.20240822
 types-simplejson==3.19.0.20240801
 types-six==1.16.21.20240513

pyright/master/requirements-pinned.txt

@@ -57,10 +57,10 @@ bitmath==1.3.3.1
 bleach==6.1.0
 blinker==1.8.2
 bokeh==3.5.2
-boto3==1.35.6
-boto3-stubs-lite==1.35.6
-botocore==1.35.6
-botocore-stubs==1.35.6
+boto3==1.35.7
+boto3-stubs-lite==1.35.7
+botocore==1.35.7
+botocore-stubs==1.35.7
 buildkite-test-collector==0.1.9
 cachecontrol==0.14.0
 cached-property==1.5.2
@@ -72,7 +72,7 @@ cattrs==23.2.3
 celery==5.4.0
 certifi==2024.7.4
 cffi==1.17.0
-cfn-lint==1.10.3
+cfn-lint==1.11.0
 chardet==5.2.0
 charset-normalizer==3.3.2
 click==8.1.7
@@ -90,7 +90,7 @@ comm==0.2.2
 configupdater==3.2
 confluent-kafka==2.5.0
 connexion==2.14.2
-contourpy==1.2.1
+contourpy==1.3.0
 coverage==7.6.1
 cron-descriptor==1.4.5
 croniter==3.0.3
@@ -175,8 +175,8 @@ dataclasses-json==0.6.7
 datadog==0.50.0
 dataproperty==1.0.1
 db-dtypes==1.3.0
-dbt-adapters==1.4.1
-dbt-common==1.7.0
+dbt-adapters==1.3.2
+dbt-common==1.3.0
 dbt-core==1.8.5
 dbt-duckdb==1.8.3
 -e examples/experimental/dagster-airlift/examples/dbt-example
@@ -185,7 +185,7 @@ dbt-semantic-interfaces==0.5.1
 debugpy==1.8.5
 decopatch==1.4.10
 decorator==5.1.1
-deepdiff==7.0.1
+deepdiff==8.0.0
 defusedxml==0.7.1
 deltalake==0.17.4
 deprecated==1.2.14
@@ -236,7 +236,7 @@ fsspec==2024.3.0
 gitdb==4.0.11
 gitpython==3.1.43
 giturlparse==0.12.0
-google-api-core==2.19.1
+google-api-core==2.19.2
 google-api-python-client==2.142.0
 google-auth==2.34.0
 google-auth-httplib2==0.2.0
@@ -247,7 +247,7 @@ google-cloud-storage==2.18.2
 google-crc32c==1.5.0
 google-re2==1.1.20240702
 google-resumable-media==2.7.2
-googleapis-common-protos==1.64.0
+googleapis-common-protos==1.65.0
 gql==3.5.0
 graphene==3.3
 graphql-core==3.2.3
@@ -266,7 +266,7 @@ html5lib==1.1
 httpcore==1.0.5
 httplib2==0.22.0
 httptools==0.6.1
-httpx==0.27.0
+httpx==0.27.2
 humanfriendly==10.0
 humanize==4.10.0
 hypothesis==6.111.2
@@ -320,7 +320,7 @@ langchain-community==0.2.9
 langchain-core==0.2.35
 langchain-openai==0.1.14
 langchain-text-splitters==0.2.2
-langsmith==0.1.104
+langsmith==0.1.106
 lazy-object-proxy==1.10.0
 leather==0.4.0
 limits==3.13.0
@@ -358,6 +358,7 @@ msgpack==1.0.8
 multidict==6.0.5
 multimethod==1.10
 mypy-boto3-ecs==1.35.2
+mypy-boto3-emr==1.35.0
 mypy-extensions==1.0.0
 mypy-protobuf==3.6.0
 mysql-connector-python==9.0.0
@@ -370,7 +371,7 @@ networkx==3.3
 nh3==0.2.18
 nodeenv==1.9.1
 noteable-origami==1.1.5
-notebook==7.2.1
+notebook==7.2.2
 notebook-shim==0.2.4
 numpy==1.26.4
 nvidia-cublas-cu12==12.1.3.1
@@ -403,6 +404,7 @@ opentelemetry-proto==1.26.0
 opentelemetry-sdk==1.26.0
 opentelemetry-semantic-conventions==0.47b0
 ordered-set==4.1.0
+orderly-set==5.2.1
 orjson==3.10.7
 outcome==1.3.0.post0
 overrides==7.7.0
@@ -467,6 +469,7 @@ pysocks==1.7.1
 pyspark==3.5.2
 pytablereader==0.31.4
 pytest==8.3.2
+pytest-asyncio==0.24.0
 pytest-cases==3.8.5
 pytest-cov==5.0.0
 pytest-mock==3.14.0
@@ -527,8 +530,8 @@ six==1.16.0
 skein==0.8.2
 skl2onnx==1.17.0
 slack-sdk==3.31.0
-sling==1.2.15
-sling-linux-amd64==1.2.15
+sling==1.2.16
+sling-linux-amd64==1.2.16
 smmap==5.0.1
 sniffio==1.3.1
 snowballstemmer==2.2.0
@@ -607,7 +610,7 @@ types-python-dateutil==2.9.0.20240821
 types-pytz==2024.1.0.20240417
 types-pyyaml==6.0.12.20240808
 types-requests==2.31.0.6
-types-s3transfer==0.10.1
+types-s3transfer==0.10.2
 types-setuptools==73.0.0.20240822
 types-simplejson==3.19.0.20240801
 types-six==1.16.21.20240513

python_modules/libraries/dagster-aws/dagster_aws/pipes/__init__.py

@@ -1,4 +1,9 @@
-from dagster_aws.pipes.clients import PipesECSClient, PipesGlueClient, PipesLambdaClient
+from dagster_aws.pipes.clients import (
+    PipesECSClient,
+    PipesEMRClient,
+    PipesGlueClient,
+    PipesLambdaClient,
+)
 from dagster_aws.pipes.context_injectors import (
     PipesLambdaEventContextInjector,
     PipesS3ContextInjector,
@@ -13,6 +18,7 @@
     "PipesGlueClient",
     "PipesLambdaClient",
     "PipesECSClient",
+    "PipesEMRClient",
     "PipesS3ContextInjector",
     "PipesLambdaEventContextInjector",
     "PipesS3MessageReader",

python_modules/libraries/dagster-aws/dagster_aws/pipes/clients/__init__.py

@@ -1,5 +1,6 @@
 from dagster_aws.pipes.clients.ecs import PipesECSClient
+from dagster_aws.pipes.clients.emr import PipesEMRClient
 from dagster_aws.pipes.clients.glue import PipesGlueClient
 from dagster_aws.pipes.clients.lambda_ import PipesLambdaClient
 
-__all__ = ["PipesGlueClient", "PipesLambdaClient", "PipesECSClient"]
+__all__ = ["PipesGlueClient", "PipesLambdaClient", "PipesECSClient", "PipesEMRClient"]