Introduce individual package publish Buildkite script (#24616)

## Summary & Motivation

Introduces a new BK pipeline which we can manually trigger to publish a
package outside of the typical release process.
Right now, pulls a list of possible packages that do not have dynamic
versions in `setup.py`:

![Screenshot 2024-09-19 at 4 08
03 PM](https://github.com/user-attachments/assets/4469c97c-d2c4-49ae-b9eb-d5887bb9a097)

The main use-case is for new integrations and experimental packages that
we may want to iterate on rapidly, e.g. release more than once a week,
separate from our stable package release process.

The pipeline:
1. Updates the version tag in code
2. Builds and publishes the package to pypi
3. Commits the new version tag
4. Pushes to a git tag `{package_name}/v{version}` and the selected
branch on GitHub

## How I Tested These Changes

Tested manually in BK

## Changelog

NOCHANGELOG

.buildkite/dagster-buildkite/dagster_buildkite/cli.py

@@ -3,6 +3,7 @@
 from dagster_buildkite.git import GitInfo
 from dagster_buildkite.pipelines.dagster_oss_main import build_dagster_oss_main_steps
 from dagster_buildkite.pipelines.dagster_oss_nightly_pipeline import build_dagster_oss_nightly_steps
+from dagster_buildkite.pipelines.prerelease_package import build_prerelease_package_steps
 from dagster_buildkite.python_packages import PythonPackages
 from dagster_buildkite.utils import buildkite_yaml_for_steps
 
@@ -24,3 +25,10 @@ def dagster_nightly() -> None:
     steps = build_dagster_oss_nightly_steps()
     buildkite_yaml = buildkite_yaml_for_steps(steps, custom_slack_channel="eng-buildkite-nightly")
     print(buildkite_yaml)  # noqa: T201
+
+
+def prerelease_package() -> None:
+    PythonPackages.load_from_git(GitInfo(directory=Path(".")))
+    steps = build_prerelease_package_steps()
+    buildkite_yaml = buildkite_yaml_for_steps(steps)
+    print(buildkite_yaml)  # noqa: T201

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/prerelease_package.py

@@ -0,0 +1,70 @@
+import re
+from pathlib import Path
+from typing import List
+
+from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import CommandStepBuilder
+from dagster_buildkite.steps.packages import _get_uncustomized_pkg_roots
+from dagster_buildkite.utils import BlockStep, BuildkiteStep
+
+
+def build_prerelease_package_steps() -> List[BuildkiteStep]:
+    steps: List[BuildkiteStep] = []
+
+    packages = (
+        _get_uncustomized_pkg_roots("python_modules", [])
+        + _get_uncustomized_pkg_roots("python_modules/libraries", [])
+        + _get_uncustomized_pkg_roots("examples/experimental", [])
+    )
+
+    # Get only packages that have a fixed version in setup.py
+    filtered_packages = []
+    for package in packages:
+        setup_file = Path(package) / "setup.py"
+        contents = setup_file.read_text()
+        if re.findall(r"version=\"[\d\.]+\"", contents):
+            filtered_packages.append(package)
+
+    input_step: BlockStep = {
+        "block": ":question: Choose package",
+        "prompt": None,
+        "fields": [
+            {
+                "select": "Select a package to publish",
+                "key": "package-to-release-path",
+                "options": [
+                    {
+                        "label": package[len("python_modules/") :]
+                        if package.startswith("python_modules/")
+                        else package,
+                        "value": package,
+                    }
+                    for package in filtered_packages
+                ],
+                "hint": None,
+                "default": None,
+                "required": True,
+                "multiple": None,
+            },
+            {
+                "text": "Enter the version to publish",
+                "required": False,
+                "key": "version-to-release",
+                "default": None,
+                "hint": "Leave blank to auto-increment the minor version",
+            },
+        ],
+    }
+    steps.append(input_step)
+
+    steps.append(
+        CommandStepBuilder(":package: Build and publish package")
+        .run(
+            "pip install build",
+            "sh ./scripts/build_and_publish.sh",
+        )
+        .on_test_image(AvailablePythonVersion.get_default(), env=["PYPI_TOKEN"])
+        .build()
+    )
+
+    return steps

.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py

@@ -74,7 +74,7 @@ def build_steps_from_package_specs(package_specs: List[PackageSpec]) -> List[Bui
 
 
 # Find packages under a root subdirectory that are not configured above.
-def _get_uncustomized_pkg_roots(root, custom_pkg_roots) -> List[str]:
+def _get_uncustomized_pkg_roots(root: str, custom_pkg_roots: List[str]) -> List[str]:
     all_files_in_root = [
         os.path.relpath(p, GIT_REPO_ROOT) for p in glob(os.path.join(GIT_REPO_ROOT, root, "*"))
     ]

.buildkite/dagster-buildkite/dagster_buildkite/utils.py

@@ -63,7 +63,40 @@ class GroupStep(TypedDict):
 
 WaitStep: TypeAlias = Literal["wait"]
 
-BuildkiteStep: TypeAlias = Union[CommandStep, GroupStep, TriggerStep, WaitStep]
+InputSelectOption = TypedDict("InputSelectOption", {"label": str, "value": str})
+InputSelectField = TypedDict(
+    "InputSelectField",
+    {
+        "select": str,
+        "key": str,
+        "options": List[InputSelectOption],
+        "hint": Optional[str],
+        "default": Optional[str],
+        "required": Optional[bool],
+        "multiple": Optional[bool],
+    },
+)
+InputTextField = TypedDict(
+    "InputTextField",
+    {
+        "text": str,
+        "key": str,
+        "hint": Optional[str],
+        "default": Optional[str],
+        "required": Optional[bool],
+    },
+)
+
+BlockStep = TypedDict(
+    "BlockStep",
+    {
+        "block": str,
+        "prompt": Optional[str],
+        "fields": List[Union[InputSelectField, InputTextField]],
+    },
+)
+
+BuildkiteStep: TypeAlias = Union[CommandStep, GroupStep, TriggerStep, WaitStep, BlockStep]
 BuildkiteLeafStep = Union[CommandStep, TriggerStep, WaitStep]
 BuildkiteTopLevelStep = Union[CommandStep, GroupStep]
 
@@ -84,7 +117,9 @@ def safe_getenv(env_var: str) -> str:
     return os.environ[env_var]
 
 
-def buildkite_yaml_for_steps(steps, custom_slack_channel: Optional[str] = None) -> str:
+def buildkite_yaml_for_steps(
+    steps: Sequence[BuildkiteStep], custom_slack_channel: Optional[str] = None
+) -> str:
     return yaml.dump(
         {
             "env": {

.buildkite/dagster-buildkite/setup.py

@@ -30,6 +30,7 @@
         "console_scripts": [
             "dagster-buildkite = dagster_buildkite.cli:dagster",
             "dagster-buildkite-nightly = dagster_buildkite.cli:dagster_nightly",
+            "dagster-buildkite-prerelease-package = dagster_buildkite.cli:prerelease_package",
         ]
     },
 )

python_modules/libraries/dagster-powerbi/setup.py

@@ -1,22 +1,10 @@
 from setuptools import find_packages, setup
 
-
-def get_version() -> str:
-    return "1!0+dev"
-    # Uncomment when ready to publish
-    # version: Dict[str, str] = {}
-    # with open(Path(__file__).parent / "dagster_powerbi/version.py", encoding="utf8") as fp:
-    #     exec(fp.read(), version)
-
-    # return version["__version__"]
-
-
-ver = get_version()
 # dont pin dev installs to avoid pip dep resolver issues
 pin = ""
 setup(
     name="dagster_powerbi",
-    version=get_version(),
+    version="0.0.3",
     author="Dagster Labs",
     author_email="[email protected]",
     license="Apache-2.0",

python_modules/libraries/dagster-sigma/dagster_sigma/__init__.py

@@ -11,6 +11,6 @@
 )
 
 # Move back to version.py and edit setup.py once we are ready to publish.
-__version__ = "1!0+dev"
+__version__ = "0.0.10"
 
 DagsterLibraryRegistry.register("dagster-sigma", __version__)

python_modules/libraries/dagster-sigma/setup.py

@@ -1,22 +1,9 @@
 from setuptools import find_packages, setup
 
-
-def get_version() -> str:
-    return "1!0+dev"
-    # Uncomment when ready to publish
-    # version: Dict[str, str] = {}
-    # with open(Path(__file__).parent / "dagster_sigma/version.py", encoding="utf8") as fp:
-    #     exec(fp.read(), version)
-
-    # return version["__version__"]
-
-
-ver = get_version()
-# dont pin dev installs to avoid pip dep resolver issues
 pin = ""
 setup(
     name="dagster_sigma",
-    version=get_version(),
+    version="0.0.10",
     author="Dagster Labs",
     author_email="[email protected]",
     license="Apache-2.0",

scripts/build_and_publish.sh

@@ -1,86 +1,51 @@
-# How to release:
-# 2. ensure you have an API key from the elementl PyPI account (account is in the password manager)
-# 3. run make adhoc_pypi from the root of the dagster-airlift directory
-# 4. once propted, use '__token__' for the username and the API key for the password
+PACKAGE_TO_RELEASE_PATH=$(buildkite-agent meta-data get package-to-release-path)
+VERSION_TO_RELEASE=$(buildkite-agent meta-data get version-to-release --default '')
 
-# Define the path to the .pypirc file
-PYPIRC_FILE="$HOME/.pypirc"
-
-PACKAGE_TO_RELEASE_PATH=$1
-VERSION_TO_RELEASE=$2
+git checkout $BUILDKITE_BRANCH
 
 if [ -z "$PACKAGE_TO_RELEASE_PATH" ]; then
     echo "Please provide the path to the package to release."
     exit 1
 fi
 if [ -z "$VERSION_TO_RELEASE" ]; then
-    echo "Please provide the version to release."
-    exit 1
-fi
-
-# Define cleanup function
-cleanup() {
-    echo "Cleaning up..."
-    rm -rf dist/*
-}
-
-# Set trap to call cleanup function on script exit
-trap cleanup EXIT
+    echo "Inferring version to release from package."
+    EXISTING_VERSION=$(grep 'version=' $PACKAGE_TO_RELEASE_PATH/setup.py)
+    echo "Existing version: $EXISTING_VERSION"
+    MAJOR_VAR=$(echo $EXISTING_VERSION | sed -E 's/.*version=[^0-9]([0-9].+)([0-9]+).*/\1/')
+    MINOR_VAR=$(echo $EXISTING_VERSION | sed -E 's/.*version=[^0-9]([0-9].+)([0-9]+).*/\2/')
+    INCREMENTED_MINOR_VAR=$((MINOR_VAR + 1))
 
-# Check if the .pypirc file exists
-if [ ! -f "$PYPIRC_FILE" ]; then
-    echo ".pypirc file not found in $HOME."
+    VERSION_TO_RELEASE="$MAJOR_VAR$INCREMENTED_MINOR_VAR"
 
-    # Prompt the user for the API token
-    read -p "Enter your API token (must start with 'pypi-'): " API_TOKEN
-
-    # Check if the API token starts with 'pypi-'
-    if [[ $API_TOKEN != pypi-* ]]; then
-        echo "Invalid API token. It must start with 'pypi-'."
-        exit 1
-    fi
-
-    # Create the .pypirc file and write the configuration
-    cat <<EOF > "$PYPIRC_FILE"
-[pypi]
-username = __token__
-password = $API_TOKEN
-EOF
-
-    echo ".pypirc file created successfully."
-else
-    echo ".pypirc file already exists in $HOME. Using that as pypi credentials."
+    echo "Going to release version $VERSION_TO_RELEASE"
 fi
 
-rm -rf dist/*
-rm -rf package_prerelease
-mkdir -p package_prerelease
-cp -R $PACKAGE_TO_RELEASE_PATH/* package_prerelease
-pushd package_prerelease
-
 # Update both a hardcoded version, if set, in setup.py, and
 # find where __version__ is set and update it
-sed -i "" "s|return \"1!0+dev\"|return \"$VERSION_TO_RELEASE\"|" setup.py
-grep -rl "__version__ = \"1!0+dev\"" ./ | xargs sed -i "" "s|\"1!0+dev\"|\"$VERSION_TO_RELEASE\"|"
+echo "Updating version in source..."
+sed -i "s|version=\".*\"|version=\"$VERSION_TO_RELEASE\"|" "$PACKAGE_TO_RELEASE_PATH/setup.py"
+grep -rl "__version__ = \".*\"" "$PACKAGE_TO_RELEASE_PATH" | xargs sed -i "s|__version__ = \".*\"|__version__ = \"$VERSION_TO_RELEASE\"|"
+
+mkdir -p package_prerelease
+cp -R $PACKAGE_TO_RELEASE_PATH/* package_prerelease
+cd package_prerelease
 
 echo "Building package..."
 python3 -m build
+
 echo "Uploading to pypi..."
-# Capture the output of the twine upload command
-TWINE_OUTPUT=$(python3 -m twine upload --repository pypi dist/* --verbose 2>&1)
-TWINE_EXIT_CODE=$?
+python3 -m twine upload --username "__token__" --password "$PYPI_TOKEN" --repository pypi dist/* --verbose
 
-# Check if the output contains a 400 error
-if echo "$TWINE_OUTPUT" | grep -q "400 Bad Request"; then
-    echo "Error: Twine upload failed with a 400 Bad Request error."
-    echo "Twine output:"
-    echo "$TWINE_OUTPUT"
-    exit 1
-elif [ $TWINE_EXIT_CODE -ne 0 ]; then
-    echo "Error: Twine upload failed with exit code $TWINE_EXIT_CODE."
-    echo "Twine output:"
-    echo "$TWINE_OUTPUT"
-    exit $TWINE_EXIT_CODE
-fi
+cd ..
+rm -rf package_prerelease
+
+echo "Committing and tagging release..."
+PACKAGE_NAME=$(echo $PACKAGE_TO_RELEASE_PATH | awk -F/ '{print $NF}')
+git add -A
+git config --global user.email "[email protected]"
+git config --global user.name "Dagster Labs"
+git commit -m "$PACKAGE_NAME $VERSION_TO_RELEASE"
 
-echo "Upload successful."
+git tag "$PACKAGE_NAME/v$VERSION_TO_RELEASE"
+git push origin "$PACKAGE_NAME/v$VERSION_TO_RELEASE"
+git push origin $BUILDKITE_BRANCH