Skip to content

Commit

Permalink
fix: set correct parallel build count in scripts (#517)
Browse files Browse the repository at this point in the history
  • Loading branch information
mattculler authored Oct 8, 2024
1 parent ab2d102 commit b508b6c
Show file tree
Hide file tree
Showing 10 changed files with 298 additions and 246 deletions.
1 change: 1 addition & 0 deletions craft_application/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -682,6 +682,7 @@ def _expand_environment(self, yaml_data: dict[str, Any], build_for: str) -> None
application_name=self.app.name, # not used in environment expansion
cache_dir=pathlib.Path(), # not used in environment expansion
arch=build_for_arch,
parallel_build_count=util.get_parallel_build_count(self.app.name),
project_name=yaml_data.get("name", ""),
project_dirs=project_dirs,
project_vars=environment_vars,
Expand Down
89 changes: 1 addition & 88 deletions craft_application/services/lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
from __future__ import annotations

import contextlib
import os
import types
from typing import TYPE_CHECKING, Any

Expand Down Expand Up @@ -200,7 +199,7 @@ def _init_lifecycle_manager(self) -> LifecycleManager:
cache_dir=self._cache_dir,
work_dir=self._work_dir,
ignore_local_sources=self._app.source_ignore_patterns,
parallel_build_count=self._get_parallel_build_count(),
parallel_build_count=util.get_parallel_build_count(self._app.name),
project_vars_part_name=self._project.adopt_info,
project_vars=self._project_vars,
track_stage_packages=True,
Expand Down Expand Up @@ -324,92 +323,6 @@ def __repr__(self) -> str:
f"{work_dir=}, {cache_dir=}, {plan=}, **{self._manager_kwargs!r})"
)

def _verify_parallel_build_count(
self, env_name: str, parallel_build_count: int | str
) -> int:
"""Verify the parallel build count is valid.
:param env_name: The name of the environment variable being checked.
:param parallel_build_count: The value of the variable.
:return: The parallel build count as an integer.
"""
try:
parallel_build_count = int(parallel_build_count)
except ValueError as err:
raise errors.InvalidParameterError(
env_name, str(os.environ[env_name])
) from err

# Ensure the value is valid positive integer
if parallel_build_count < 1:
raise errors.InvalidParameterError(env_name, str(parallel_build_count))

return parallel_build_count

def _get_parallel_build_count(self) -> int:
"""Get the number of parallel builds to run.
The parallel build count is determined by the first available of the
following environment variables in the order:
- <APP_NAME>_PARALLEL_BUILD_COUNT
- CRAFT_PARALLEL_BUILD_COUNT
- <APP_NAME>_MAX_PARALLEL_BUILD_COUNT
- CRAFT_MAX_PARALLEL_BUILD_COUNT
where the MAX_PARALLEL_BUILD_COUNT variables are dynamically compared to
the number of CPUs, and the smaller of the two is used.
If no environment variable is set, the CPU count is used.
If the CPU count is not available for some reason, 1 is used as a fallback.
"""
parallel_build_count = None

# fixed parallel build count environment variable
for env_name in [
(self._app.name + "_PARALLEL_BUILD_COUNT").upper(),
"CRAFT_PARALLEL_BUILD_COUNT",
]:
if os.environ.get(env_name):
parallel_build_count = self._verify_parallel_build_count(
env_name, os.environ[env_name]
)
emit.debug(
f"Using parallel build count of {parallel_build_count} "
f"from environment variable {env_name!r}"
)
break

# CPU count related max parallel build count environment variable
if parallel_build_count is None:
cpu_count = os.cpu_count() or 1
for env_name in [
(self._app.name + "_MAX_PARALLEL_BUILD_COUNT").upper(),
"CRAFT_MAX_PARALLEL_BUILD_COUNT",
]:
if os.environ.get(env_name):
parallel_build_count = min(
cpu_count,
self._verify_parallel_build_count(
env_name, os.environ[env_name]
),
)
emit.debug(
f"Using parallel build count of {parallel_build_count} "
f"from environment variable {env_name!r}"
)
break

# Default to CPU count if no max environment variable is set
if parallel_build_count is None:
parallel_build_count = cpu_count
emit.debug(
f"Using parallel build count of {parallel_build_count} "
"from CPU count"
)

return parallel_build_count

def _get_local_keys_path(self) -> Path | None:
"""Return a directory with public keys for package-repositories.
Expand Down
2 changes: 2 additions & 0 deletions craft_application/util/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
is_running_from_snap,
)
from craft_application.util.string import humanize_list, strtobool
from craft_application.util.system import get_parallel_build_count
from craft_application.util.yaml import dump_yaml, safe_yaml_load

__all__ = [
Expand All @@ -52,4 +53,5 @@
"dump_yaml",
"safe_yaml_load",
"retry",
"get_parallel_build_count",
]
105 changes: 105 additions & 0 deletions craft_application/util/system.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# This file is part of craft-application.
#
# Copyright 2024 Canonical Ltd.
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY,
# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
"""System-level util functions."""
from __future__ import annotations

import os

from craft_cli import emit

from craft_application.errors import InvalidParameterError


def _verify_parallel_build_count(env_name: str, parallel_build_count: int | str) -> int:
"""Verify the parallel build count is valid.
:param env_name: The name of the environment variable being checked.
:param parallel_build_count: The value of the variable.
:return: The parallel build count as an integer.
"""
try:
parallel_build_count = int(parallel_build_count)
except ValueError as err:
raise InvalidParameterError(env_name, str(os.environ[env_name])) from err

# Ensure the value is valid positive integer
if parallel_build_count < 1:
raise InvalidParameterError(env_name, str(parallel_build_count))

return parallel_build_count


def get_parallel_build_count(app_name: str) -> int:
"""Get the number of parallel builds to run.
The parallel build count is determined by the first available of the
following environment variables in the order:
- <APP_NAME>_PARALLEL_BUILD_COUNT
- CRAFT_PARALLEL_BUILD_COUNT
- <APP_NAME>_MAX_PARALLEL_BUILD_COUNT
- CRAFT_MAX_PARALLEL_BUILD_COUNT
where the MAX_PARALLEL_BUILD_COUNT variables are dynamically compared to
the number of CPUs, and the smaller of the two is used.
If no environment variable is set, the CPU count is used.
If the CPU count is not available for some reason, 1 is used as a fallback.
"""
parallel_build_count = None

# fixed parallel build count environment variable
for env_name in [
(app_name + "_PARALLEL_BUILD_COUNT").upper(),
"CRAFT_PARALLEL_BUILD_COUNT",
]:
if os.environ.get(env_name):
parallel_build_count = _verify_parallel_build_count(
env_name, os.environ[env_name]
)
emit.debug(
f"Using parallel build count of {parallel_build_count} "
f"from environment variable {env_name!r}"
)
break

# CPU count related max parallel build count environment variable
if parallel_build_count is None:
cpu_count = os.cpu_count() or 1
for env_name in [
(app_name + "_MAX_PARALLEL_BUILD_COUNT").upper(),
"CRAFT_MAX_PARALLEL_BUILD_COUNT",
]:
if os.environ.get(env_name):
parallel_build_count = min(
cpu_count,
_verify_parallel_build_count(env_name, os.environ[env_name]),
)
emit.debug(
f"Using parallel build count of {parallel_build_count} "
f"from environment variable {env_name!r}"
)
break

# Default to CPU count if no max environment variable is set
if parallel_build_count is None:
parallel_build_count = cpu_count
emit.debug(
f"Using parallel build count of {parallel_build_count} "
"from CPU count"
)

return parallel_build_count
5 changes: 5 additions & 0 deletions docs/reference/changelog.rst
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@
Changelog
*********

X.Y.Z (yyyy-mmm-dd)
-------------------

- Fix: set CRAFT_PARALLEL_BUILD_COUNT correctly in ``override-`` scripts.

4.2.6 (2024-Oct-04)
-------------------

Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ docs = [
"sphinx-lint==0.9.1",
]
apt = [
"python-apt>=2.4.0;sys_platform=='linux'"
"python-apt>=2.4.0;sys_platform=='linux'",
]

[build-system]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,6 @@ parts:
echo "project_version: \"${CRAFT_PROJECT_VERSION}\"" >> $target_file
echo "arch_build_for: \"${CRAFT_ARCH_BUILD_FOR}\"" >> $target_file
echo "arch_triplet_build_for: \"${CRAFT_ARCH_TRIPLET_BUILD_FOR}\"" >> $target_file
echo "arch_build_on: \"${CRAFT_ARCH_BUILD_ON}\"" >> $target_file
echo "arch_triplet_build_on: \"${CRAFT_ARCH_TRIPLET_BUILD_ON}\"" >> $target_file
echo "arch_build_on: \"${CRAFT_ARCH_BUILD_ON}\"" >> $target_file
echo "arch_triplet_build_on: \"${CRAFT_ARCH_TRIPLET_BUILD_ON}\"" >> $target_file
echo "parallel_build_count: \"${CRAFT_PARALLEL_BUILD_COUNT}\"" >> $target_file
5 changes: 5 additions & 0 deletions tests/integration/test_application.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,10 @@ def test_global_environment(
],
)

# Check that this odd value makes its way through to the yaml build script
build_count = "5"
mocker.patch.dict("os.environ", {"TESTCRAFT_PARALLEL_BUILD_COUNT": build_count})

# Run in destructive mode
monkeypatch.setattr(
"sys.argv", ["testcraft", "prime", "--destructive-mode", *arguments]
Expand All @@ -328,6 +332,7 @@ def test_global_environment(
assert variables["arch_triplet_build_on"].startswith(
util.convert_architecture_deb_to_platform(util.get_host_architecture())
)
assert variables["parallel_build_count"] == build_count


@pytest.fixture
Expand Down
Loading

0 comments on commit b508b6c

Please sign in to comment.