Merge branch 'master' into patch-2

.buildkite/dagster-buildkite/dagster_buildkite/cli.py

@@ -2,9 +2,10 @@
 
 from dagster_buildkite.git import GitInfo
 from dagster_buildkite.pipelines.dagster_oss_main import build_dagster_oss_main_steps
+from dagster_buildkite.pipelines.dagster_oss_nightly_pipeline import build_dagster_oss_nightly_steps
+from dagster_buildkite.pipelines.prerelease_package import build_prerelease_package_steps
 from dagster_buildkite.python_packages import PythonPackages
-
-from .utils import buildkite_yaml_for_steps
+from dagster_buildkite.utils import buildkite_yaml_for_steps
 
 CLI_HELP = """This CLI is used for generating Buildkite YAML. Each function corresponds to an entry
 point defined in `setup.py`. Buildkite invokes these entry points when loading the specification for
@@ -17,3 +18,17 @@ def dagster() -> None:
     steps = build_dagster_oss_main_steps()
     buildkite_yaml = buildkite_yaml_for_steps(steps)
     print(buildkite_yaml)  # noqa: T201
+
+
+def dagster_nightly() -> None:
+    PythonPackages.load_from_git(GitInfo(directory=Path(".")))
+    steps = build_dagster_oss_nightly_steps()
+    buildkite_yaml = buildkite_yaml_for_steps(steps, custom_slack_channel="eng-buildkite-nightly")
+    print(buildkite_yaml)  # noqa: T201
+
+
+def prerelease_package() -> None:
+    PythonPackages.load_from_git(GitInfo(directory=Path(".")))
+    steps = build_prerelease_package_steps()
+    buildkite_yaml = buildkite_yaml_for_steps(steps)
+    print(buildkite_yaml)  # noqa: T201

.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py

@@ -9,11 +9,10 @@
 
 from dagster_buildkite.git import ChangedFiles
 from dagster_buildkite.python_packages import PythonPackages, changed_filetypes
-
-from .python_version import AvailablePythonVersion
-from .step_builder import BuildkiteQueue
-from .steps.tox import build_tox_step
-from .utils import (
+from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import BuildkiteQueue
+from dagster_buildkite.steps.tox import build_tox_step
+from dagster_buildkite.utils import (
     BuildkiteLeafStep,
     BuildkiteTopLevelStep,
     GroupStep,
@@ -165,9 +164,9 @@ def build_steps(self) -> List[BuildkiteTopLevelStep]:
                 pytest_python_versions = sorted(
                     list(set(default_python_versions) - set(unsupported_python_versions))
                 )
-                # Use lowest supported python version if no defaults match.
+                # Use highest supported python version if no defaults_match
                 if len(pytest_python_versions) == 0:
-                    pytest_python_versions = [supported_python_versions[0]]
+                    pytest_python_versions = [supported_python_versions[-1]]
 
                 for py_version in pytest_python_versions:
                     version_factor = AvailablePythonVersion.to_tox_factor(py_version)

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_main.py

@@ -10,7 +10,7 @@
 )
 from dagster_buildkite.steps.docs import build_docs_steps
 from dagster_buildkite.steps.trigger import build_trigger_step
-from dagster_buildkite.utils import BuildkiteStep, is_release_branch, safe_getenv
+from dagster_buildkite.utils import BuildkiteStep, is_release_branch, message_contains, safe_getenv
 
 
 def build_dagster_oss_main_steps() -> List[BuildkiteStep]:
@@ -33,8 +33,11 @@ def build_dagster_oss_main_steps() -> List[BuildkiteStep]:
             pipeline_name = "oss-internal-compatibility"
             trigger_branch = _get_setting("INTERNAL_BRANCH") or "master"
             async_step = False
-            # Use OSS_COMPAT_SLIM by default unless an internal branch is explicitly specified
-            oss_compat_slim = _get_setting("OSS_COMPAT_SLIM") or not _get_setting("INTERNAL_BRANCH")
+            # Use OSS_COMPAT_SLIM by default unless an internal branch is explicitly specified or
+            # the commit message contains "NO_SKIP"
+            oss_compat_slim = _get_setting("OSS_COMPAT_SLIM") or not (
+                _get_setting("INTERNAL_BRANCH") or message_contains("NO_SKIP")
+            )
 
         steps.append(
             build_trigger_step(

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py

@@ -0,0 +1,48 @@
+from typing import List
+
+from dagster_buildkite.package_spec import PackageSpec
+from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.steps.packages import (
+    build_steps_from_package_specs,
+    gcp_creds_extra_cmds,
+    k8s_extra_cmds,
+)
+from dagster_buildkite.utils import BuildkiteStep
+
+
+def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
+    steps: List[BuildkiteStep] = []
+
+    steps += build_steps_from_package_specs(
+        [
+            PackageSpec(
+                "python_modules/libraries/dagster-dbt",
+                pytest_tox_factors=["dbt18-snowflake", "dbt18-bigquery"],
+                env_vars=[
+                    "SNOWFLAKE_ACCOUNT",
+                    "SNOWFLAKE_USER",
+                    "SNOWFLAKE_PASSWORD",
+                    "GCP_PROJECT_ID",
+                ],
+                pytest_extra_cmds=gcp_creds_extra_cmds,
+                unsupported_python_versions=[
+                    AvailablePythonVersion.V3_12,
+                ],
+            ),
+            PackageSpec(
+                "python_modules/libraries/dagster-k8s",
+                env_vars=[
+                    "AWS_ACCOUNT_ID",
+                    "AWS_ACCESS_KEY_ID",
+                    "AWS_SECRET_ACCESS_KEY",
+                    "BUILDKITE_SECRETS_BUCKET",
+                ],
+                pytest_tox_factors=[
+                    "nightly",
+                ],
+                pytest_extra_cmds=k8s_extra_cmds,
+            ),
+        ]
+    )
+
+    return steps

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py

@@ -0,0 +1,70 @@
+import re
+from pathlib import Path
+from typing import List
+
+from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import CommandStepBuilder
+from dagster_buildkite.steps.packages import _get_uncustomized_pkg_roots
+from dagster_buildkite.utils import BlockStep, BuildkiteStep
+
+
+def build_prerelease_package_steps() -> List[BuildkiteStep]:
+    steps: List[BuildkiteStep] = []
+
+    packages = (
+        _get_uncustomized_pkg_roots("python_modules", [])
+        + _get_uncustomized_pkg_roots("python_modules/libraries", [])
+        + _get_uncustomized_pkg_roots("examples/experimental", [])
+    )
+
+    # Get only packages that have a fixed version in setup.py
+    filtered_packages = []
+    for package in packages:
+        setup_file = Path(package) / "setup.py"
+        contents = setup_file.read_text()
+        if re.findall(r"version=\"[\d\.]+\"", contents):
+            filtered_packages.append(package)
+
+    input_step: BlockStep = {
+        "block": ":question: Choose package",
+        "prompt": None,
+        "fields": [
+            {
+                "select": "Select a package to publish",
+                "key": "package-to-release-path",
+                "options": [
+                    {
+                        "label": package[len("python_modules/") :]
+                        if package.startswith("python_modules/")
+                        else package,
+                        "value": package,
+                    }
+                    for package in filtered_packages
+                ],
+                "hint": None,
+                "default": None,
+                "required": True,
+                "multiple": None,
+            },
+            {
+                "text": "Enter the version to publish",
+                "required": False,
+                "key": "version-to-release",
+                "default": None,
+                "hint": "Leave blank to auto-increment the minor version",
+            },
+        ],
+    }
+    steps.append(input_step)
+
+    steps.append(
+        CommandStepBuilder(":package: Build and publish package")
+        .run(
+            "pip install build",
+            "sh ./scripts/build_and_publish.sh",
+        )
+        .on_test_image(AvailablePythonVersion.get_default(), env=["PYPI_TOKEN"])
+        .build()
+    )
+
+    return steps

.buildkite/dagster-buildkite/dagster_buildkite/python_packages.py

@@ -1,30 +1,59 @@
+# pyright: reportUnnecessaryTypeIgnoreComment=false
+
 import logging
+import subprocess
 from distutils import core as distutils_core
 from importlib import reload
 from pathlib import Path
 from typing import Dict, Optional, Set
 
-import pathspec
+import tomli
 from pkg_resources import Requirement, parse_requirements
 
 from dagster_buildkite.git import ChangedFiles, GitInfo
 
 changed_filetypes = [".py", ".cfg", ".toml", ".yaml", ".ipynb", ".yml", ".ini", ".jinja"]
 
 
+def _path_is_relative_to(p: Path, u: Path) -> bool:
+    # see https://docs.python.org/3/library/pathlib.html#pathlib.PurePath.is_relative_to
+    return u == p or u in p.parents
+
+
 class PythonPackage:
-    def __init__(self, setup_py_path: Path):
-        self.directory = setup_py_path.parent
+    def __init__(self, setup_path: Path):
+        self.directory = setup_path
 
-        # run_setup stores state in a global variable. Reload the module
-        # each time we use it - otherwise we'll get the previous invocation's
-        # distribution if our setup.py doesn't implement setup() correctly
-        reload(distutils_core)
-        distribution = distutils_core.run_setup(str(setup_py_path), stop_after="init")
+        if (setup_path / "setup.py").exists():
+            # run_setup stores state in a global variable. Reload the module
+            # each time we use it - otherwise we'll get the previous invocation's
+            # distribution if our setup.py doesn't implement setup() correctly
+            reload(distutils_core)
 
-        self._install_requires = distribution.install_requires  # type: ignore[attr-defined]
-        self._extras_require = distribution.extras_require  # type: ignore[attr-defined]
-        self.name = distribution.get_name()
+            distribution = distutils_core.run_setup(str(setup_path), stop_after="init")
+
+            self._install_requires = distribution.install_requires  # type: ignore[attr-defined]
+            self._extras_require = distribution.extras_require  # type: ignore[attr-defined]
+            self.name = distribution.get_name()
+        else:
+            pyproject_toml = setup_path / "pyproject.toml"
+            assert (
+                pyproject_toml.exists()
+            ), f"expected pyproject.toml to exist in directory {setup_path}"
+
+            try:
+                with open(pyproject_toml, "rb") as f:
+                    project = tomli.load(f)["project"]
+            except KeyError:
+                # this directory has a pyproject.toml but isn't really a python projects,
+                # ie docs/
+                self.name = setup_path.name
+                self._install_requires = []
+                self._extras_require = {}
+            else:
+                self.name = project["name"]
+                self._install_requires = project["dependnecies"]
+                self._extras_require = project.get("optional-dependencies", {})
 
     @property
     def install_requires(self) -> Set[Requirement]:
@@ -112,22 +141,18 @@ def load_from_git(cls, git_info: GitInfo) -> None:
 
         logging.info("Finding Python packages:")
 
-        git_ignore = git_info.directory / ".gitignore"
-
-        if git_ignore.exists():
-            ignored = git_ignore.read_text().splitlines()
-            git_ignore_spec = pathspec.PathSpec.from_lines("gitwildmatch", ignored)
-        else:
-            git_ignore_spec = pathspec.PathSpec([])
-
         # Consider any setup.py file to be a package
-        packages = set(
-            [
-                PythonPackage(Path(setup))
-                for setup in git_info.directory.rglob("setup.py")
-                if not git_ignore_spec.match_file(str(setup))
-            ]
-        )
+        output = subprocess.check_output(
+            ["git", "ls-files", "."],
+            cwd=str(git_info.directory),
+        ).decode("utf-8")
+        packages = []
+        for file in output.split("\n"):
+            path = git_info.directory / Path(file)
+            if path.is_dir() and (
+                (path / "setup.py").exists() or (path / "pyproject.toml").exists()
+            ):
+                packages.append(PythonPackage(path))
 
         for package in sorted(packages):
             logging.info("  - " + package.name)
@@ -140,7 +165,7 @@ def load_from_git(cls, git_info: GitInfo) -> None:
             for change in ChangedFiles.all:
                 if (
                     # Our change is in this package's directory
-                    (change in package.directory.rglob("*"))
+                    _path_is_relative_to(change, package.directory)
                     # The file can alter behavior - exclude things like README changes
                     and (change.suffix in changed_filetypes)
                     # The file is not part of a test suite. We treat this differently

.buildkite/dagster-buildkite/dagster_buildkite/python_version.py

@@ -11,6 +11,7 @@ class AvailablePythonVersion(str, Enum):
     V3_9 = "3.9"
     V3_10 = "3.10"
     V3_11 = "3.11"
+    V3_12 = "3.12"
 
     @classmethod
     def get_all(cls) -> List["AvailablePythonVersion"]:

.buildkite/dagster-buildkite/dagster_buildkite/step_builder.py

@@ -2,14 +2,14 @@
 from enum import Enum
 from typing import Dict, List, Optional
 
-from .images.versions import BUILDKITE_TEST_IMAGE_VERSION
-from .python_version import AvailablePythonVersion
-from .utils import CommandStep, safe_getenv
+from dagster_buildkite.images.versions import BUILDKITE_TEST_IMAGE_VERSION
+from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.utils import CommandStep, safe_getenv
 
 DEFAULT_TIMEOUT_IN_MIN = 25
 
-DOCKER_PLUGIN = "docker#v3.7.0"
-ECR_PLUGIN = "ecr#v2.2.0"
+DOCKER_PLUGIN = "docker#v5.10.0"
+ECR_PLUGIN = "ecr#v2.7.0"
 
 
 AWS_ACCOUNT_ID = os.environ.get("AWS_ACCOUNT_ID")
@@ -42,6 +42,8 @@ def __init__(
             "retry": {
                 "automatic": [
                     {"exit_status": -1, "limit": 2},  # agent lost
+                    {"exit_status": 143, "limit": 2},  # agent lost
+                    {"exit_status": 2, "limit": 2},  # often a uv read timeout
                     {"exit_status": 255, "limit": 2},  # agent forced shut down
                 ],
                 "manual": {"permit_on_passed": True},
@@ -57,8 +59,8 @@ def run(self, *commands: str) -> "CommandStepBuilder":
     def _base_docker_settings(self) -> Dict[str, object]:
         return {
             "shell": ["/bin/bash", "-xeuc"],
-            "always-pull": True,
             "mount-ssh-agent": True,
+            "mount-buildkite-agent": True,
         }
 
     def on_python_image(self, image: str, env: Optional[List[str]] = None) -> "CommandStepBuilder":