Skip to content

Commit

Permalink
add dagster-ext as dep of dagster (#16703)
Browse files Browse the repository at this point in the history
  • Loading branch information
jamiedemaria authored Sep 22, 2023
1 parent b824aa3 commit 9be3eab
Show file tree
Hide file tree
Showing 84 changed files with 88 additions and 1 deletion.
1 change: 1 addition & 0 deletions docs/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ deps =

# Can't stub deps because processed by sphinx-click
-e ../python_modules/dagster
-e ../python_modules/dagster-ext
-e ../python_modules/dagster-graphql
-e ../python_modules/dagster-webserver
-e ../python_modules/libraries/dagster-celery
Expand Down
1 change: 1 addition & 0 deletions examples/assets_dbt_python/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
source: -e ../../python_modules/dagster-webserver
Expand Down
1 change: 1 addition & 0 deletions examples/assets_dynamic_partitions/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/libraries/dagster-duckdb/
-e ../../python_modules/libraries/dagster-duckdb-pandas/
-e .
Expand Down
1 change: 1 addition & 0 deletions examples/assets_modern_data_stack/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
source: -e ../../python_modules/dagster-graphql
Expand Down
1 change: 1 addition & 0 deletions examples/assets_pandas_pyspark/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-spark
Expand Down
1 change: 1 addition & 0 deletions examples/assets_pandas_type_metadata/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/dagster[test]
Expand Down
1 change: 1 addition & 0 deletions examples/assets_smoke_test/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-pandas/
Expand Down
1 change: 1 addition & 0 deletions examples/deploy_docker/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* DEPLOY_DOCKER_WEBSERVER_HOST
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
allowlist_externals =
/bin/bash
commands =
Expand Down
1 change: 1 addition & 0 deletions examples/deploy_ecs/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* DEPLOY_DOCKER_WEBSERVER_HOST
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-test
allowlist_externals =
/bin/bash
Expand Down
1 change: 1 addition & 0 deletions examples/deploy_k8s/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-aws
-e ../../python_modules/libraries/dagster-celery
Expand Down
1 change: 1 addition & 0 deletions examples/development_to_production/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e .[test]
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
Expand Down
1 change: 1 addition & 0 deletions examples/docs_snippets/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN POSTGRES_TEST_DB_HOST BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-graphql
-e ../../python_modules/dagster-webserver
-e ../../python_modules/libraries/dagstermill[test]
Expand Down
1 change: 1 addition & 0 deletions examples/feature_graph_backed_assets/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e .
Expand Down
1 change: 1 addition & 0 deletions examples/project_fully_featured/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_PASSWORD
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-pandas/
Expand Down
1 change: 1 addition & 0 deletions examples/quickstart_aws/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
source: -e ../../python_modules/libraries/dagster-aws/
Expand Down
1 change: 1 addition & 0 deletions examples/quickstart_etl/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
-e .
Expand Down
1 change: 1 addition & 0 deletions examples/quickstart_gcp/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
source: -e ../../python_modules/libraries/dagster-gcp
Expand Down
1 change: 1 addition & 0 deletions examples/quickstart_snowflake/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
; note: "source" does not work at this time due to dagster-cloud source access
deps =
source: -e ../../python_modules/dagster[test]
source: -e ../../python_modules/dagster-ext
pypi: dagster[test]
pypi: -r ../temp_pins.txt
source: -e ../../python_modules/libraries/dagster-gcp
Expand Down
1 change: 1 addition & 0 deletions examples/with_airflow/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-airflow[test]
Expand Down
1 change: 1 addition & 0 deletions examples/with_great_expectations/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-pandas
Expand Down
1 change: 1 addition & 0 deletions examples/with_pyspark/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-spark
Expand Down
1 change: 1 addition & 0 deletions examples/with_pyspark_emr/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-aws[test]
Expand Down
1 change: 1 addition & 0 deletions examples/with_wandb/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-webserver
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-wandb
Expand Down
1 change: 1 addition & 0 deletions helm/dagster/schema/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-ext
-e ../../../python_modules/libraries/dagster-pandas
-e ../../../python_modules/libraries/dagster-aws
-e ../../../python_modules/libraries/dagster-azure
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-ext
-e ../../../python_modules/dagster-graphql
-e ../../../python_modules/dagster-test
-e ../../../python_modules/libraries/dagster-k8s
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* BACKCOMPAT_TESTS_WEBSERVER_HOST EARLIEST_TESTED_RELEASE

deps =
-e ../../../python_modules/dagster-ext
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-webserver
-e ../../../python_modules/dagster-graphql
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ WORKDIR /tmp

RUN pip install \
-e python_modules/dagster \
-e python_modules/dagster-graphql \
-e python_modules/dagster-ext \
-e python_modules/dagster-graphql \
-e python_modules/libraries/dagster-postgres \
-e python_modules/libraries/dagster-docker

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ WORKDIR /tmp

RUN pip install \
-e python_modules/dagster \
-e python_modules/dagster-ext \
-e python_modules/dagster-graphql \
-e python_modules/dagster-webserver \
-e python_modules/libraries/dagster-postgres \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ alias copy_py="rsync -av \
--exclude .coverage"

copy_py $ROOT/python_modules/dagster \
$ROOT/python_modules/dagster-ext \
$ROOT/python_modules/dagit \
$ROOT/python_modules/dagster-webserver \
$ROOT/python_modules/dagster-graphql \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = HOME AIRFLOW_HOME AWS_* BUILDKITE* CI_* COVERALLS_REPO_TOKEN DAGSTER_* DOCKER_* GOOGLE_* KUBECONFIG
deps =
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-ext
-e ../../../python_modules/dagster-graphql
-e ../../../python_modules/dagster-test
-e ../../../python_modules/libraries/dagster-pandas
Expand Down
1 change: 1 addition & 0 deletions integration_tests/test_suites/daemon-test-suite/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ passenv = HOME AIRFLOW_HOME AWS_* BUILDKITE* CI_* COVERALLS_REPO_TOKEN DAGSTER_*
deps =
objgraph
-e ../../../python_modules/dagster[test]
-e ../../../python_modules/dagster-ext
-e ../../../python_modules/dagster-graphql
-e ../../../python_modules/dagit
-e ../../../python_modules/dagster-webserver
Expand Down
1 change: 1 addition & 0 deletions js_modules/dagster-ui/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ setenv =
STRICT_GRPC_SERVER_PROCESS_WAIT = "1"
deps =
-e ../../python_modules/dagster[test]
-e ../../python_modules/dagster-ext
-e ../../python_modules/dagster-graphql
-e ../../python_modules/libraries/dagster-slack
-e ../../python_modules/dagster-webserver[starlette]
Expand Down
1 change: 1 addition & 0 deletions python_modules/dagit/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ setenv =
deps =
objgraph
-e ../dagster[test]
-e ../dagster-ext
-e ../dagster-graphql
-e ../dagster-webserver
-e .
Expand Down
1 change: 1 addition & 0 deletions python_modules/dagster-ext/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ setenv =
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../dagster[test]
-e ../dagster-ext
-e ../libraries/dagster-aws
-e .[test]

Expand Down
1 change: 1 addition & 0 deletions python_modules/dagster-graphql/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ setenv =
STRICT_GRPC_SERVER_PROCESS_WAIT = "1"
deps =
-e ../dagster[test]
-e ../dagster-ext
postgres: -e ../libraries/dagster-postgres
-e .
allowlist_externals =
Expand Down
1 change: 1 addition & 0 deletions python_modules/dagster-test/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../dagster[test]
-e ../dagster-ext
-e ../dagster-graphql
-e ../libraries/dagster-pandas
-e ../libraries/dagster-aws
Expand Down
1 change: 1 addition & 0 deletions python_modules/dagster-webserver/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ setenv =
deps =
objgraph
-e ../dagster[test]
-e ../dagster-ext
-e ../dagster-graphql
-e .[notebook,test]

Expand Down
4 changes: 4 additions & 0 deletions python_modules/dagster/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ def get_version() -> str:
# grpcio 1.44.0 is the min version compatible with both protobuf 3 and 4
GRPC_VERSION_FLOOR = "1.44.0"

ver = get_version()
# dont pin dev installs to avoid pip dep resolver issues
pin = "" if ver == "1!0+dev" else f"=={ver}"

setup(
name="dagster",
Expand Down Expand Up @@ -104,6 +107,7 @@ def get_version() -> str:
"universal_pathlib",
# https://github.com/pydantic/pydantic/issues/5821
"pydantic != 1.10.7,<2.0.0",
f"dagster-ext-process{pin}",
],
extras_require={
"docker": ["docker"],
Expand Down
1 change: 1 addition & 0 deletions python_modules/dagster/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ deps =
general_tests_old_protobuf: protobuf<4
-e ../dagster-test
-e .[mypy,test,pyright]
-e ../dagster-ext
allowlist_externals =
/bin/bash
commands =
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-airbyte/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ skipsdist = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-test
-e ../dagster-managed-elements
-e .[test]
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-airflow/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ setenv =
passenv = HOME AIRFLOW_HOME AWS_* BUILDKITE* CI_* COVERALLS_REPO_TOKEN DAGSTER_* DOCKER_* GOOGLE_* KUBECONFIG, POSTGRES_TEST_DB_HOST
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-test
-e ../dagster-aws
-e ../dagster-pandas
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-aws/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN AWS_* BUILDKITE* SSH_*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../dagster-spark
-e ../dagster-pyspark
-e .[redshift,test]
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-azure/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN AZURE_* BUILDKITE* SSH_*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../dagster-spark
-e ../dagster-pyspark
-e .
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-celery-docker/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN GOOGLE_APPLICATION_CREDENTIALS BUILDKITE* AWS_SECRET_ACCESS_KEY AWS_ACCESS_KEY_ID DAGSTER_DOCKER_* POSTGRES_TEST_DB_HOST
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-graphql
-e ../../dagster-test
-e ../dagster-aws
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-celery-k8s/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-graphql
-e ../../dagster-test
-e ../dagster-pandas
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-celery/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = HOME CI_PULL_REQUEST COVERALLS_REPO_TOKEN DASK_ADDRESS AWS_* BUILDKITE* DAGSTER_*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-graphql
-e ../../dagster-test
-e ../dagster-postgres
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-census/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ skipsdist = true
passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e .
whitelist_externals =
/bin/bash
Expand Down
1 change: 1 addition & 0 deletions python_modules/libraries/dagster-dask/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ download = True
passenv = CI_PULL_REQUEST COVERALLS_REPO_TOKEN DASK_ADDRESS AWS_* BUILDKITE*
deps =
-e ../../dagster[test]
-e ../../dagster-ext
-e ../../dagster-graphql
-e ../dagster-aws
-e ../dagster-pandas
Expand Down
Loading

1 comment on commit 9be3eab

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagster-docs ready!

✅ Preview
https://dagster-docs-67694zx7o-elementl.vercel.app
https://master.dagster.dagster-docs.io

Built with commit 9be3eab.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.