Use Python 3.12 in benchmarks (#1215)

I originally used Python 3.10, since 3.10 and 3.11 are by far the most
common (at least for [Ruff](https://pypistats.org/packages/ruff)). But
3.12 should give Python tools the most favorable benchmarks.

BENCHMARKS.md

@@ -1,6 +1,7 @@
 # Benchmarks
 
-All benchmarks were computed on macOS, and come with a few important caveats:
+All benchmarks were computed on macOS using Python 3.12 (for non-Puffin tools), and come with a few
+important caveats:
 
 - Benchmark performance may vary dramatically across different operating systems and filesystems.
   In particular, Puffin uses different installation strategies based on the underlying filesystem's
@@ -25,31 +26,31 @@ Benchmarking package installation (e.g., `puffin pip sync`) with a warm cache. T
 to removing and recreating a virtual environment, and then populating it with dependencies that
 you've installed previously on the same machine.
 
-![](https://github.com/astral-sh/ruff/assets/1309177/6ceea7aa-4813-4ea8-8c95-b8013d702cf4)
+![install-warm](https://github.com/astral-sh/ruff/assets/1309177/b6cb8d48-52e0-45c2-ae15-0a3f69ec3263)
 
 ## Cold Installation
 
 Benchmarking package installation (e.g., `puffin pip sync`) with a cold cache. This is equivalent
 to running `puffin pip sync` on a new machine or in CI (assuming that the package manager cache is
 not shared across runs).
 
-![](https://github.com/astral-sh/ruff/assets/1309177/c960d6fd-ec34-467e-9aa2-d4e6713abed0)
+![install-cold](https://github.com/astral-sh/ruff/assets/1309177/ed86c193-582f-4163-b369-f12ec3905c3c)
 
 ## Warm Resolution
 
 Benchmarking dependency resolution (e.g., `puffin pip compile`) with a warm cache, but no existing
 lockfile. This is equivalent to blowing away an existing `requirements.txt` file to regenerate it
 from a `requirements.in` file.
 
-![](https://github.com/astral-sh/ruff/assets/1309177/aab99181-e54e-4bdb-9ce6-15b018ef8466)
+![resolve-warm](https://github.com/astral-sh/ruff/assets/1309177/a4ca9d23-1148-4103-abe7-a35fa488409d)
 
 ## Cold Resolution
 
 Benchmarking dependency resolution (e.g., `puffin pip compile`) with a cold cache. This is
 equivalent to running `puffin pip compile` on a new machine or in CI (assuming that the package
 manager cache is not shared across runs).
 
-![](https://github.com/astral-sh/ruff/assets/1309177/a6075ebc-bb8f-46db-a3b4-14ee5f713565)
+![resolve-cold](https://github.com/astral-sh/ruff/assets/1309177/556ac7aa-0a6a-4f94-b0d9-90b25461de7b)
 
 ## Reproduction
 

crates/puffin-dev/src/render_benchmarks.rs

@@ -23,13 +23,15 @@ pub(crate) fn render_benchmarks(args: &RenderBenchmarksArgs) -> Result<()> {
     // but we assume we're running over a single benchmark here.)
     for result in &mut results.results {
         if result.command.starts_with("puffin") {
-            result.command = "puffin".into();
+            result.command = "Puffin".into();
         } else if result.command.starts_with("pip-compile") {
             result.command = "pip-compile".into();
         } else if result.command.starts_with("pip-sync") {
             result.command = "pip-sync".into();
         } else if result.command.starts_with("poetry") {
-            result.command = "poetry".into();
+            result.command = "Poetry".into();
+        } else if result.command.starts_with("pdm") {
+            result.command = "PDM".into();
         } else {
             return Err(anyhow!("unknown command: {}", result.command));
         }

scripts/bench/__main__.py

@@ -4,7 +4,7 @@
 installed, and that a Puffin release builds exists at `./target/release/puffin`
 (relative to the repository root).
 
-This script assumes that Python 3.10 is installed.
+This script assumes that Python 3.12 is installed.
 
 To set up the required environment, run:
 
@@ -300,7 +300,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
-            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}",
+            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
             command=[
                 self.path,
                 os.path.abspath(requirements_file),
@@ -317,7 +317,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
             command=[
                 self.path,
                 os.path.abspath(requirements_file),
@@ -357,9 +357,11 @@ def setup(self, requirements_file: str, *, cwd: str) -> None:
                 "bench",
                 "--no-interaction",
                 "--python",
-                ">=3.10",
+                "3.12",
             ],
             cwd=cwd,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
         )
 
         # Parse the pyproject.toml.
@@ -518,7 +520,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
                 f"rm -rf {config_dir} && "
                 f"rm -rf {cache_dir} && "
                 f"rm -rf {data_dir} &&"
-                f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
             ),
             command=[
                 f"POETRY_CONFIG_DIR={config_dir}",
@@ -558,7 +560,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
             command=[
                 f"POETRY_CONFIG_DIR={config_dir}",
                 f"POETRY_CACHE_DIR={cache_dir}",
@@ -595,8 +597,10 @@ def setup(self, requirements_file: str, *, cwd: str) -> None:
 
         # Create a PDM project.
         subprocess.check_call(
-            [self.path, "init", "--non-interactive", "--python", "3.10"],
+            [self.path, "init", "--non-interactive", "--python", "3.12"],
             cwd=cwd,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
         )
 
         # Parse the pyproject.toml.
@@ -635,7 +639,7 @@ def resolve_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
         cache_dir = os.path.join(cwd, "cache", "pdm")
 
         return Command(
-            name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})",
+            name=f"{self.name} ({Benchmark.RESOLVE_WARM.value})",
             prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}",
             command=[
                 self.path,
@@ -719,7 +723,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
             prepare=(
                 f"rm -rf {cache_dir} && "
                 f"{self.path} config cache_dir {cache_dir} && "
-                f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
             ),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
@@ -750,10 +754,10 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
         cache_dir = os.path.join(cwd, "cache", "pdm")
 
         return Command(
-            name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
+            name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
             prepare=(
                 f"{self.path} config cache_dir {cache_dir} && "
-                f"virtualenv --clear -p 3.10 {venv_dir} --no-seed"
+                f"virtualenv --clear -p 3.12 {venv_dir} --no-seed"
             ),
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
@@ -871,7 +875,7 @@ def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_COLD.value})",
-            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.10 {venv_dir}",
+            prepare=f"rm -rf {cache_dir} && virtualenv --clear -p 3.12 {venv_dir}",
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
                 self.path,
@@ -889,7 +893,7 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None:
 
         return Command(
             name=f"{self.name} ({Benchmark.INSTALL_WARM.value})",
-            prepare=f"virtualenv --clear -p 3.10 {venv_dir}",
+            prepare=f"virtualenv --clear -p 3.12 {venv_dir}",
             command=[
                 f"VIRTUAL_ENV={venv_dir}",
                 self.path,