diff --git a/scripts/bench/__main__.py b/scripts/bench/__main__.py index 97c92cc95703..0a7e8e5da69d 100644 --- a/scripts/bench/__main__.py +++ b/scripts/bench/__main__.py @@ -574,6 +574,197 @@ def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: ) +class Pdm(Suite): + def __init__(self, path: str | None = None) -> None: + self.name = path or "pdm" + self.path = path or "pdm" + + def setup(self, requirements_file: str, *, cwd: str) -> None: + """Initialize a PDM project from a requirements file.""" + import tomli + import tomli_w + from packaging.requirements import Requirement + + # Parse all dependencies from the requirements file. + with open(requirements_file) as fp: + requirements = [ + Requirement(line) + for line in fp + if not line.lstrip().startswith("#") and len(line.strip()) > 0 + ] + + # Create a PDM project. + subprocess.check_call( + [self.path, "init", "--non-interactive", "--python", "3.10"], + cwd=cwd, + ) + + # Parse the pyproject.toml. + with open(os.path.join(cwd, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["project"]["dependencies"] = [ + str(requirement) for requirement in requirements + ] + + with open(os.path.join(cwd, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + def resolve_cold(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", + prepare=f"rm -rf {cache_dir} && rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--project", + cwd, + ], + ) + + def resolve_warm(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_COLD.value})", + prepare=f"rm -rf {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--project", + cwd, + ], + ) + + def resolve_incremental( + self, requirements_file: str, *, cwd: str + ) -> Command | None: + import tomli + import tomli_w + + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + # Add a dependency to the requirements file. + with open(os.path.join(cwd, "pyproject.toml"), "rb") as fp: + pyproject = tomli.load(fp) + + # Add the dependencies to the pyproject.toml. + pyproject["project"]["dependencies"] += [INCREMENTAL_REQUIREMENT] + + with open(os.path.join(cwd, "pyproject.toml"), "wb") as fp: + tomli_w.dump(pyproject, fp) + + # Store the baseline lock file. + baseline = os.path.join(cwd, "baseline.lock") + shutil.copyfile(pdm_lock, baseline) + + pdm_lock = os.path.join(cwd, "pdm.lock") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.RESOLVE_INCREMENTAL.value})", + prepare=f"rm -f {pdm_lock} && cp {baseline} {pdm_lock} && {self.path} config cache_dir {cache_dir}", + command=[ + self.path, + "lock", + "--update-reuse", + "--project", + cwd, + ], + ) + + def install_cold(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + venv_dir = os.path.join(cwd, ".venv") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", + prepare=( + f"rm -rf {cache_dir} && " + f"{self.path} config cache_dir {cache_dir} && " + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + ), + command=[ + f"VIRTUAL_ENV={venv_dir}", + self.path, + "sync", + "--project", + cwd, + ], + ) + + def install_warm(self, requirements_file: str, *, cwd: str) -> Command | None: + self.setup(requirements_file, cwd=cwd) + + pdm_lock = os.path.join(cwd, "pdm.lock") + assert not os.path.exists(pdm_lock), f"Lock file already exists at: {pdm_lock}" + + # Run a resolution, to ensure that the lock file exists. + # TODO(charlie): Make this a `setup`. + subprocess.check_call( + [self.path, "lock"], + cwd=cwd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + assert os.path.exists(pdm_lock), f"Lock file doesn't exist at: {pdm_lock}" + + venv_dir = os.path.join(cwd, ".venv") + cache_dir = os.path.join(cwd, "cache", "pdm") + + return Command( + name=f"{self.name} ({Benchmark.INSTALL_COLD.value})", + prepare=( + f"{self.path} config cache_dir {cache_dir} && " + f"virtualenv --clear -p 3.10 {venv_dir} --no-seed" + ), + command=[ + f"VIRTUAL_ENV={venv_dir}", + self.path, + "sync", + "--project", + cwd, + ], + ) + + class Puffin(Suite): def __init__(self, *, path: str | None = None) -> Command | None: """Initialize a Puffin benchmark.""" @@ -727,9 +918,7 @@ def main(): parser.add_argument( "--verbose", "-v", action="store_true", help="Print verbose output." ) - parser.add_argument( - "--json", action="store_true", help="Export results to JSON." - ) + parser.add_argument("--json", action="store_true", help="Export results to JSON.") parser.add_argument( "--warmup", type=int, @@ -765,6 +954,11 @@ def main(): help="Whether to benchmark Poetry (requires Poetry to be installed).", action="store_true", ) + parser.add_argument( + "--pdm", + help="Whether to benchmark PDM (requires PDM to be installed).", + action="store_true", + ) parser.add_argument( "--puffin", help="Whether to benchmark Puffin (assumes a Puffin binary exists at `./target/release/puffin`).", @@ -788,6 +982,12 @@ def main(): help="Path(s) to the Poetry binary to benchmark.", action="append", ) + parser.add_argument( + "--pdm-path", + type=str, + help="Path(s) to the PDM binary to benchmark.", + action="append", + ) parser.add_argument( "--puffin-path", type=str, @@ -819,6 +1019,8 @@ def main(): suites.append(PipCompile()) if args.poetry: suites.append(Poetry()) + if args.pdm: + suites.append(Pdm()) if args.puffin: suites.append(Puffin()) for path in args.pip_sync_path or []: @@ -827,6 +1029,8 @@ def main(): suites.append(PipCompile(path=path)) for path in args.poetry_path or []: suites.append(Poetry(path=path)) + for path in args.pdm_path or []: + suites.append(Pdm(path=path)) for path in args.puffin_path or []: suites.append(Puffin(path=path)) diff --git a/scripts/bench/requirements.in b/scripts/bench/requirements.in index b86f70e819cd..db0650e5c149 100644 --- a/scripts/bench/requirements.in +++ b/scripts/bench/requirements.in @@ -1,3 +1,4 @@ +pdm pip-tools poetry tomli diff --git a/scripts/bench/requirements.txt b/scripts/bench/requirements.txt index a0654ce35875..415c21a48da6 100644 --- a/scripts/bench/requirements.txt +++ b/scripts/bench/requirements.txt @@ -1,13 +1,19 @@ -# This file was autogenerated by Puffin v0.0.1 via the following command: -# puffin pip compile ./scripts/requirements.in --python-version 3.10 +# This file was autogenerated by Puffin v0.0.3 via the following command: +# puffin pip compile scripts/bench/requirements.in -o scripts/bench/requirements.txt +blinker==1.7.0 + # via pdm build==1.0.3 # via # pip-tools # poetry cachecontrol==0.13.1 - # via poetry + # via + # pdm + # poetry certifi==2023.11.17 - # via requests + # via + # pdm + # requests cffi==1.16.0 # via xattr charset-normalizer==3.3.2 @@ -20,6 +26,8 @@ crashtest==0.4.1 # via # cleo # poetry +dep-logic==0.0.4 + # via pdm distlib==0.3.8 # via virtualenv dulwich==0.21.7 @@ -27,17 +35,25 @@ dulwich==0.21.7 fastjsonschema==2.19.1 # via poetry filelock==3.13.1 - # via virtualenv + # via + # cachecontrol + # virtualenv +findpython==0.4.1 + # via pdm idna==3.6 # via requests -importlib-metadata==7.0.1 - # via keyring installer==0.7.0 - # via poetry + # via + # pdm + # poetry jaraco-classes==3.3.0 # via keyring keyring==24.3.0 # via poetry +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py more-itertools==10.1.0 # via jaraco-classes msgpack==1.0.7 @@ -45,7 +61,12 @@ msgpack==1.0.7 packaging==23.2 # via # build + # dep-logic + # findpython + # pdm # poetry + # unearth +pdm==2.12.2 pexpect==4.9.0 # via poetry pip==23.3.2 @@ -55,6 +76,7 @@ pkginfo==1.9.6 # via poetry platformdirs==3.11.0 # via + # pdm # poetry # virtualenv poetry==1.7.1 @@ -69,10 +91,15 @@ ptyprocess==0.7.0 # via pexpect pycparser==2.21 # via cffi +pygments==2.17.2 + # via rich pyproject-hooks==1.0.0 # via # build + # pdm # poetry +python-dotenv==1.0.1 + # via pdm rapidfuzz==3.6.1 # via cleo requests==2.31.0 @@ -80,32 +107,42 @@ requests==2.31.0 # cachecontrol # poetry # requests-toolbelt + # unearth requests-toolbelt==1.0.0 - # via poetry + # via + # pdm + # poetry +resolvelib==1.0.1 + # via pdm +rich==13.7.0 + # via pdm setuptools==69.0.3 # via pip-tools shellingham==1.5.4 - # via poetry -tomli==2.0.1 # via - # build - # pip-tools + # pdm # poetry - # pyproject-hooks +tomli==2.0.1 tomli-w==1.0.0 tomlkit==0.12.3 - # via poetry + # via + # pdm + # poetry trove-classifiers==2023.11.29 # via poetry +truststore==0.8.0 + # via pdm +unearth==0.14.0 + # via pdm urllib3==2.1.0 # via # dulwich # requests virtualenv==20.25.0 - # via poetry + # via + # pdm + # poetry wheel==0.42.0 # via pip-tools xattr==0.10.1 # via poetry -zipp==3.17.0 - # via importlib-metadata