Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(toolchain): use bazel to extract zstd archives #2412

Merged
merged 2 commits into from
Nov 18, 2024
Merged

refactor(toolchain): use bazel to extract zstd archives #2412

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ee2f98d
refactor(toolchain): use bazel to extract `zstd` archives
aignas Nov 15, 2024
02e008e
Merge branch 'main' into chore/use-bazel-zstd-support
rickeylev Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ Unreleased changes template.

{#v0-0-0-changed}
### Changed
* Nothing yet.
* (python_repository) Start honouring the `strip_prefix` field for `zstd` archives.

{#v0-0-0-fixed}
### Fixed
Expand All @@ -66,6 +66,11 @@ Unreleased changes template.
### Removed
* (publish) Remove deprecated `requirements.txt` for the `twine` dependencies.
Please use `requirements_linux.txt` instead.
* (python_repository) Use bazel's built in `zstd` support and remove attributes
for customizing the `zstd` binary to be used for `zstd` archives in the
{bzl:obj}`python_repository` repository_rule. This affects the
{bzl:obj}`python_register_toolchains` and
{bzl:obj}`python_register_multi_toolchains` callers in the `WORKSPACE`.

{#v0-39-0}
## [0.39.0] - 2024-11-13
Expand Down
2 changes: 1 addition & 1 deletion python/private/python.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ def _get_toolchain_config(*, modules, _fail = fail):
"strip_prefix": {
platform: item["strip_prefix"]
for platform in item["sha256"]
},
} if type(item["strip_prefix"]) == type("") else item["strip_prefix"],
"url": {
platform: [item["url"]]
for platform in item["sha256"]
Expand Down
61 changes: 10 additions & 51 deletions python/private/python_repository.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"""This file contains repository rules and macros to support toolchain registration.
"""

load("//python:versions.bzl", "FREETHREADED", "PLATFORMS")
load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS")
load(":auth.bzl", "get_auth")
load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
load(":text_util.bzl", "render")
Expand Down Expand Up @@ -72,51 +72,13 @@ def _python_repository_impl(rctx):
urls = rctx.attr.urls or [rctx.attr.url]
auth = get_auth(rctx, urls)

if release_filename.endswith(".zst"):
rctx.download(
if INSTALL_ONLY in release_filename:
rctx.download_and_extract(
url = urls,
sha256 = rctx.attr.sha256,
output = release_filename,
stripPrefix = rctx.attr.strip_prefix,
auth = auth,
)
unzstd = rctx.which("unzstd")
if not unzstd:
url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version)
rctx.download_and_extract(
url = url,
sha256 = rctx.attr.zstd_sha256,
auth = auth,
)
working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version)

repo_utils.execute_checked(
rctx,
op = "python_repository.MakeZstd",
arguments = [
repo_utils.which_checked(rctx, "make"),
"--jobs=4",
],
timeout = 600,
quiet = True,
working_directory = working_directory,
logger = logger,
)
zstd = "{working_directory}/zstd".format(working_directory = working_directory)
unzstd = "./unzstd"
rctx.symlink(zstd, unzstd)

repo_utils.execute_checked(
rctx,
op = "python_repository.ExtractRuntime",
arguments = [
repo_utils.which_checked(rctx, "tar"),
"--extract",
"--strip-components=2",
"--use-compress-program={unzstd}".format(unzstd = unzstd),
"--file={}".format(release_filename),
],
logger = logger,
)
else:
rctx.download_and_extract(
url = urls,
Expand All @@ -125,6 +87,12 @@ def _python_repository_impl(rctx):
auth = auth,
)

# Strip the things that are not present in the INSTALL_ONLY builds
# NOTE: if the dirs are not present, we will not fail here
rctx.delete("python/build")
rctx.delete("python/licenses")
rctx.delete("python/PYTHON.json")

patches = rctx.attr.patches
if patches:
for patch in patches:
Expand Down Expand Up @@ -378,15 +346,6 @@ function defaults (e.g. `single_version_override` for `MODULE.bazel` files.
"urls": attr.string_list(
doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
),
"zstd_sha256": attr.string(
default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
),
"zstd_url": attr.string(
default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz",
),
"zstd_version": attr.string(
default = "1.5.2",
),
"_rule_name": attr.string(default = "python_repository"),
},
environ = [REPO_DEBUG_ENV_VAR],
Expand Down
22 changes: 19 additions & 3 deletions python/versions.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ MACOS_NAME = "mac os"
LINUX_NAME = "linux"
WINDOWS_NAME = "windows"
FREETHREADED = "freethreaded"
INSTALL_ONLY = "install_only"

DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download"

Expand Down Expand Up @@ -52,7 +53,7 @@ TOOL_VERSIONS = {
"x86_64-apple-darwin": "8d06bec08db8cdd0f64f4f05ee892cf2fcbc58cfb1dd69da2caab78fac420238",
"x86_64-unknown-linux-gnu": "aec8c4c53373b90be7e2131093caa26063be6d9d826f599c935c0e1042af3355",
},
"strip_prefix": "python",
"strip_prefix": "python/install",
},
"3.8.12": {
"url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz",
Expand Down Expand Up @@ -579,7 +580,22 @@ TOOL_VERSIONS = {
"x86_64-pc-windows-msvc-freethreaded": "bfd89f9acf866463bc4baf01733da5e767d13f5d0112175a4f57ba91f1541310",
"x86_64-unknown-linux-gnu-freethreaded": "a73adeda301ad843cce05f31a2d3e76222b656984535a7b87696a24a098b216c",
},
"strip_prefix": "python",
"strip_prefix": {
"aarch64-apple-darwin": "python",
"aarch64-unknown-linux-gnu": "python",
"ppc64le-unknown-linux-gnu": "python",
"s390x-unknown-linux-gnu": "python",
"x86_64-apple-darwin": "python",
"x86_64-pc-windows-msvc": "python",
"x86_64-unknown-linux-gnu": "python",
"aarch64-apple-darwin-freethreaded": "python/install",
"aarch64-unknown-linux-gnu-freethreaded": "python/install",
"ppc64le-unknown-linux-gnu-freethreaded": "python/install",
"s390x-unknown-linux-gnu-freethreaded": "python/install",
"x86_64-apple-darwin-freethreaded": "python/install",
"x86_64-pc-windows-msvc-freethreaded": "python/install",
"x86_64-unknown-linux-gnu-freethreaded": "python/install",
},
},
}

Expand Down Expand Up @@ -777,7 +793,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U
}[p],
)
else:
build = "install_only"
build = INSTALL_ONLY

if WINDOWS_NAME in platform:
build = "shared-" + build
Expand Down