From ee2f98d7f7c913eab825bbf7030ed0f0003fae4a Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:31:35 +0900 Subject: [PATCH] refactor(toolchain): use bazel to extract `zstd` archives Here we remove the dependence on the `zstd` archive downloaded from GH for extracting `zstd` toolchains because bazel now supports `zstd` extraction for a long time. This mainly cleans up the code which is again used a lot more due to #2386. --- CHANGELOG.md | 7 +++- python/private/python.bzl | 2 +- python/private/python_repository.bzl | 61 +++++----------------------- python/versions.bzl | 22 ++++++++-- 4 files changed, 36 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c351487605..30b1dc5fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,7 +52,7 @@ Unreleased changes template. {#v0-0-0-changed} ### Changed -* Nothing yet. +* (python_repository) Start honouring the `strip_prefix` field for `zstd` archives. {#v0-0-0-fixed} ### Fixed @@ -66,6 +66,11 @@ Unreleased changes template. ### Removed * (publish) Remove deprecated `requirements.txt` for the `twine` dependencies. Please use `requirements_linux.txt` instead. +* (python_repository) Use bazel's built in `zstd` support and remove attributes + for customizing the `zstd` binary to be used for `zstd` archives in the + {bzl:obj}`python_repository` repository_rule. This affects the + {bzl:obj}`python_register_toolchains` and + {bzl:obj}`python_register_multi_toolchains` callers in the `WORKSPACE`. {#v0-39-0} ## [0.39.0] - 2024-11-13 diff --git a/python/private/python.bzl b/python/private/python.bzl index 8632554c51..ec6f73e41f 100644 --- a/python/private/python.bzl +++ b/python/private/python.bzl @@ -462,7 +462,7 @@ def _get_toolchain_config(*, modules, _fail = fail): "strip_prefix": { platform: item["strip_prefix"] for platform in item["sha256"] - }, + } if type(item["strip_prefix"]) == type("") else item["strip_prefix"], "url": { platform: [item["url"]] for platform in item["sha256"] diff --git a/python/private/python_repository.bzl b/python/private/python_repository.bzl index 9ffa196a20..c7407c8f2c 100644 --- a/python/private/python_repository.bzl +++ b/python/private/python_repository.bzl @@ -15,7 +15,7 @@ """This file contains repository rules and macros to support toolchain registration. """ -load("//python:versions.bzl", "FREETHREADED", "PLATFORMS") +load("//python:versions.bzl", "FREETHREADED", "INSTALL_ONLY", "PLATFORMS") load(":auth.bzl", "get_auth") load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":text_util.bzl", "render") @@ -72,51 +72,13 @@ def _python_repository_impl(rctx): urls = rctx.attr.urls or [rctx.attr.url] auth = get_auth(rctx, urls) - if release_filename.endswith(".zst"): - rctx.download( + if INSTALL_ONLY in release_filename: + rctx.download_and_extract( url = urls, sha256 = rctx.attr.sha256, - output = release_filename, + stripPrefix = rctx.attr.strip_prefix, auth = auth, ) - unzstd = rctx.which("unzstd") - if not unzstd: - url = rctx.attr.zstd_url.format(version = rctx.attr.zstd_version) - rctx.download_and_extract( - url = url, - sha256 = rctx.attr.zstd_sha256, - auth = auth, - ) - working_directory = "zstd-{version}".format(version = rctx.attr.zstd_version) - - repo_utils.execute_checked( - rctx, - op = "python_repository.MakeZstd", - arguments = [ - repo_utils.which_checked(rctx, "make"), - "--jobs=4", - ], - timeout = 600, - quiet = True, - working_directory = working_directory, - logger = logger, - ) - zstd = "{working_directory}/zstd".format(working_directory = working_directory) - unzstd = "./unzstd" - rctx.symlink(zstd, unzstd) - - repo_utils.execute_checked( - rctx, - op = "python_repository.ExtractRuntime", - arguments = [ - repo_utils.which_checked(rctx, "tar"), - "--extract", - "--strip-components=2", - "--use-compress-program={unzstd}".format(unzstd = unzstd), - "--file={}".format(release_filename), - ], - logger = logger, - ) else: rctx.download_and_extract( url = urls, @@ -125,6 +87,12 @@ def _python_repository_impl(rctx): auth = auth, ) + # Strip the things that are not present in the INSTALL_ONLY builds + # NOTE: if the dirs are not present, we will not fail here + rctx.delete("python/build") + rctx.delete("python/licenses") + rctx.delete("python/PYTHON.json") + patches = rctx.attr.patches if patches: for patch in patches: @@ -378,15 +346,6 @@ function defaults (e.g. `single_version_override` for `MODULE.bazel` files. "urls": attr.string_list( doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.", ), - "zstd_sha256": attr.string( - default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", - ), - "zstd_url": attr.string( - default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", - ), - "zstd_version": attr.string( - default = "1.5.2", - ), "_rule_name": attr.string(default = "python_repository"), }, environ = [REPO_DEBUG_ENV_VAR], diff --git a/python/versions.bzl b/python/versions.bzl index 774c24d1b9..688c4e2ceb 100644 --- a/python/versions.bzl +++ b/python/versions.bzl @@ -20,6 +20,7 @@ MACOS_NAME = "mac os" LINUX_NAME = "linux" WINDOWS_NAME = "windows" FREETHREADED = "freethreaded" +INSTALL_ONLY = "install_only" DEFAULT_RELEASE_BASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download" @@ -52,7 +53,7 @@ TOOL_VERSIONS = { "x86_64-apple-darwin": "8d06bec08db8cdd0f64f4f05ee892cf2fcbc58cfb1dd69da2caab78fac420238", "x86_64-unknown-linux-gnu": "aec8c4c53373b90be7e2131093caa26063be6d9d826f599c935c0e1042af3355", }, - "strip_prefix": "python", + "strip_prefix": "python/install", }, "3.8.12": { "url": "20220227/cpython-{python_version}+20220227-{platform}-{build}.tar.gz", @@ -579,7 +580,22 @@ TOOL_VERSIONS = { "x86_64-pc-windows-msvc-freethreaded": "bfd89f9acf866463bc4baf01733da5e767d13f5d0112175a4f57ba91f1541310", "x86_64-unknown-linux-gnu-freethreaded": "a73adeda301ad843cce05f31a2d3e76222b656984535a7b87696a24a098b216c", }, - "strip_prefix": "python", + "strip_prefix": { + "aarch64-apple-darwin": "python", + "aarch64-unknown-linux-gnu": "python", + "ppc64le-unknown-linux-gnu": "python", + "s390x-unknown-linux-gnu": "python", + "x86_64-apple-darwin": "python", + "x86_64-pc-windows-msvc": "python", + "x86_64-unknown-linux-gnu": "python", + "aarch64-apple-darwin-freethreaded": "python/install", + "aarch64-unknown-linux-gnu-freethreaded": "python/install", + "ppc64le-unknown-linux-gnu-freethreaded": "python/install", + "s390x-unknown-linux-gnu-freethreaded": "python/install", + "x86_64-apple-darwin-freethreaded": "python/install", + "x86_64-pc-windows-msvc-freethreaded": "python/install", + "x86_64-unknown-linux-gnu-freethreaded": "python/install", + }, }, } @@ -777,7 +793,7 @@ def get_release_info(platform, python_version, base_url = DEFAULT_RELEASE_BASE_U }[p], ) else: - build = "install_only" + build = INSTALL_ONLY if WINDOWS_NAME in platform: build = "shared-" + build