From 51e8609ee83b2eedcf150431972d18d79ed68de5 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Wed, 31 Jan 2024 12:51:13 -0800 Subject: [PATCH] Use Python 3.12 in benchmarks (#1215) I originally used Python 3.10, since 3.10 and 3.11 are by far the most common (at least for [Ruff](https://pypistats.org/packages/ruff)). But 3.12 should give Python tools the most favorable benchmarks. --- BENCHMARKS.md | 11 ++++---- crates/puffin-dev/src/render_benchmarks.rs | 6 +++-- scripts/bench/__main__.py | 30 ++++++++++++---------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 47ddf3d37e92..fb2dc4923f03 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -1,6 +1,7 @@ # Benchmarks -All benchmarks were computed on macOS, and come with a few important caveats: +All benchmarks were computed on macOS using Python 3.12 (for non-Puffin tools), and come with a few +important caveats: - Benchmark performance may vary dramatically across different operating systems and filesystems. In particular, Puffin uses different installation strategies based on the underlying filesystem's @@ -25,7 +26,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a warm cache. T to removing and recreating a virtual environment, and then populating it with dependencies that you've installed previously on the same machine. -![](https://github.com/astral-sh/ruff/assets/1309177/6ceea7aa-4813-4ea8-8c95-b8013d702cf4) +![install-warm](https://github.com/astral-sh/ruff/assets/1309177/b6cb8d48-52e0-45c2-ae15-0a3f69ec3263) ## Cold Installation @@ -33,7 +34,7 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a cold cache. T to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is not shared across runs). -![](https://github.com/astral-sh/ruff/assets/1309177/c960d6fd-ec34-467e-9aa2-d4e6713abed0) +![install-cold](https://github.com/astral-sh/ruff/assets/1309177/ed86c193-582f-4163-b369-f12ec3905c3c) ## Warm Resolution @@ -41,7 +42,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a warm cach lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it from a `requirements.in` file. -![](https://github.com/astral-sh/ruff/assets/1309177/aab99181-e54e-4bdb-9ce6-15b018ef8466) +![resolve-warm](https://github.com/astral-sh/ruff/assets/1309177/a4ca9d23-1148-4103-abe7-a35fa488409d) ## Cold Resolution @@ -49,7 +50,7 @@ Benchmarking dependency resolution (e.g., `puffin pip compile`) with a cold cach equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package manager cache is not shared across runs). -![](https://github.com/astral-sh/ruff/assets/1309177/a6075ebc-bb8f-46db-a3b4-14ee5f713565) +![resolve-cold](https://github.com/astral-sh/ruff/assets/1309177/556ac7aa-0a6a-4f94-b0d9-90b25461de7b) ## Reproduction diff --git a/crates/puffin-dev/src/render_benchmarks.rs b/crates/puffin-dev/src/render_benchmarks.rs index 55ebb4c6105d..3c3982a0ae20 100644 --- a/crates/puffin-dev/src/render_benchmarks.rs +++ b/crates/puffin-dev/src/render_benchmarks.rs @@ -23,13 +23,15 @@ pub(crate) fn render_benchmarks(args: &RenderBenchmarksArgs) -> Result<()> { // but we assume we're running over a single benchmark here.) for result in &mut results.results { if result.command.starts_with("puffin") { - result.command = "puffin".into(); + result.command = "Puffin".into(); } else if result.command.starts_with("pip-compile") { result.command = "pip-compile".into(); } else if result.command.starts_with("pip-sync") { result.command = "pip-sync".into(); } else if result.command.starts_with("poetry") { - result.command = "poetry".into(); + result.command = "Poetry".into(); + } else if result.command.starts_with("pdm") { + result.command = "PDM".into(); } else { return Err(anyhow!("unknown command: {}", result.command)); } diff --git a/scripts/bench/__main__.py b/scripts/bench/__main__.py index 0a7e8e5da69d..a3ab72c62934 100644 --- a/scripts/bench/__main__.py +++ b/scripts/bench/__main__.py @@ -4,7 +4,7 @@ installed, and that a Puffin release builds exists at `./target/release/puffin` (relative to the repository root). -This script assumes that Python 3.10 is installed. +This script assumes that Python 3.12 is installed. To set up the required environment, run: @@ -300,7 +300,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: return Command( name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", - prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}", + prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}", command=[ self.path, os.path.abspath(requirements_file), @@ -317,7 +317,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: return Command( name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", - prepare=f"virtualenv --clear -p 3.10 {venv_dir}", + prepare=f"virtualenv --clear -p 3.12 {venv_dir}", command=[ self.path, os.path.abspath(requirements_file), @@ -357,9 +357,11 @@ def setup(self, requirements_file: str, *, cwd: str) -> None: "bench", "--no-interaction", "--python", - ">=3.10", + "3.12", ], cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) # Parse the pyproject.toml. @@ -518,7 +520,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: f"rm -rf {config_dir} && " f"rm -rf {cache_dir} && " f"rm -rf {data_dir} &&" - f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + f"virtualenv --clear -p 3.12 {venv_dir} --no-seed" ), command=[ f"POETRY_CONFIG_DIR={config_dir}", @@ -558,7 +560,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: return Command( name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", - prepare=f"virtualenv --clear -p 3.10 {venv_dir}", + prepare=f"virtualenv --clear -p 3.12 {venv_dir}", command=[ f"POETRY_CONFIG_DIR={config_dir}", f"POETRY_CACHE_DIR={cache_dir}", @@ -595,8 +597,10 @@ def setup(self, requirements_file: str, *, cwd: str) -> None: # Create a PDM project. subprocess.check_call( - [self.path, "init", "--non-interactive", "--python", "3.10"], + [self.path, "init", "--non-interactive", "--python", "3.12"], cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) # Parse the pyproject.toml. @@ -635,7 +639,7 @@ def resolve_warm(self, requirements_file: str, *, cwd: str) -> Command | None: cache_dir = os.path.join(cwd, "cache", "pdm") return Command( - name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", + name=f"{self.name} ({Benchmark.RESOLVE_WARM.value})", prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", command=[ self.path, @@ -719,7 +723,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: prepare=( f"rm -rf {cache_dir} && " f"{self.path} config cache_dir {cache_dir} && " - f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + f"virtualenv --clear -p 3.12 {venv_dir} --no-seed" ), command=[ f"VIRTUAL_ENV={venv_dir}", @@ -750,10 +754,10 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: cache_dir = os.path.join(cwd, "cache", "pdm") return Command( - name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", + name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", prepare=( f"{self.path} config cache_dir {cache_dir} && " - f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + f"virtualenv --clear -p 3.12 {venv_dir} --no-seed" ), command=[ f"VIRTUAL_ENV={venv_dir}", @@ -871,7 +875,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: return Command( name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", - prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}", + prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}", command=[ f"VIRTUAL_ENV={venv_dir}", self.path, @@ -889,7 +893,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: return Command( name=f"{self.name} ({Benchmark.INSTALL_WARM.value})", - prepare=f"virtualenv --clear -p 3.10 {venv_dir}", + prepare=f"virtualenv --clear -p 3.12 {venv_dir}", command=[ f"VIRTUAL_ENV={venv_dir}", self.path,