simplify parallel jobs (#10068)

* simplify parallel jobs * add missing files * import multiprocessing * review * fix tests * fix tests * fix tests
conan-io · Dec 1, 2021 · bc83dfa · bc83dfa
1 parent cbc9586
commit bc83dfa
Show file tree

Hide file tree

Showing 20 changed files with 99 additions and 274 deletions.
diff --git a/.gitignore b/.gitignore
@@ -105,4 +105,5 @@ cacert.pem
 
 # add excluded
 !conans/client/build
+!conan/tools/build
 !conans/test/unittests/client/build
diff --git a/conan/tools/build/__init__.py b/conan/tools/build/__init__.py
@@ -0,0 +1 @@
+from conan.tools.build.cpu import build_jobs
diff --git a/conan/tools/build/cpu.py b/conan/tools/build/cpu.py
@@ -0,0 +1,30 @@
+import math
+import multiprocessing
+
+from conans.util.files import load
+
+
+def build_jobs(conanfile):
+    njobs = conanfile.conf["tools.build:jobs"]
+    if njobs is not None:
+        result = int(njobs)
+        if result == 0:
+            return None
+        return result
+    return _cpu_count()
+
+
+def _cpu_count():
+    try:
+        try:
+            # This is necessary to deduce docker cpu_count
+            cfs_quota_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_quota_us"))
+            cfs_period_us = int(load("/sys/fs/cgroup/cpu/cpu.cfs_period_us"))
+            if cfs_quota_us > 0 and cfs_period_us > 0:
+                return int(math.ceil(cfs_quota_us / cfs_period_us))
+        except (OSError, TypeError):
+            pass
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        # print("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu")
+        return 1  # Safe guess
diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py
@@ -1,14 +1,11 @@
 import os
 import platform
 
+from conan.tools.build import build_jobs
 from conan.tools.cmake.utils import is_multi_configuration
 from conan.tools.files import load_toolchain_args, chdir, mkdir
-from conan.tools.gnu.make import make_jobs_cmd_line_arg
-from conan.tools.meson.meson import ninja_jobs_cmd_line_arg
-from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg, \
-    msbuild_max_cpu_count_cmd_line_arg
-from conans.client import tools
-from conans.client.tools.oss import cpu_count, args_to_string
+from conan.tools.microsoft.msbuild import msbuild_verbosity_cmd_line_arg
+from conans.client.tools.oss import args_to_string
 from conans.errors import ConanException
 
 
@@ -19,27 +16,19 @@ def _validate_recipe(conanfile):
                              " or 'cmake_find_package_multi' generators")
 
 
-def _cmake_cmd_line_args(conanfile, generator, parallel):
+def _cmake_cmd_line_args(conanfile, generator):
     args = []
     if not generator:
         return args
 
     # Arguments related to parallel
-    if parallel:
-        if "Makefiles" in generator and "NMake" not in generator:
-            njobs = make_jobs_cmd_line_arg(conanfile)
-            if njobs:
-                args.append(njobs)
-
-        if "Ninja" in generator and "NMake" not in generator:
-            njobs = ninja_jobs_cmd_line_arg(conanfile)
-            if njobs:
-                args.append(njobs)
-
-        if "Visual Studio" in generator:
-            max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(conanfile)
-            if max_cpu_count:
-                args.append(max_cpu_count)
+    njobs = build_jobs(conanfile)
+    if njobs and ("Makefiles" in generator or "Ninja" in generator) and "NMake" not in generator:
+        args.append("-j{}".format(njobs))
+
+    maxcpucount = conanfile.conf["tools.microsoft.msbuild:max_cpu_count"]
+    if maxcpucount and "Visual Studio" in generator:
+        args.append("/m:{}".format(njobs))
 
     # Arguments for verbosity
     if "Visual Studio" in generator:
@@ -57,12 +46,11 @@ class CMake(object):
     are passed to the command line, plus the ``--config Release`` for builds in multi-config
     """
 
-    def __init__(self, conanfile, parallel=True, namespace=None):
+    def __init__(self, conanfile, namespace=None):
         _validate_recipe(conanfile)
 
         # Store a reference to useful data
         self._conanfile = conanfile
-        self._parallel = parallel
         self._namespace = namespace
 
         toolchain_file_content = load_toolchain_args(self._conanfile.generators_folder, namespace=self._namespace)
@@ -72,10 +60,6 @@ def __init__(self, conanfile, parallel=True, namespace=None):
         self._cmake_program = "cmake"  # Path to CMake should be handled by environment
 
     def configure(self, build_script_folder=None):
-        # TODO: environment?
-        if not self._conanfile.should_configure:
-            return
-
         cmakelist_folder = self._conanfile.source_folder
         if build_script_folder:
             cmakelist_folder = os.path.join(self._conanfile.source_folder, build_script_folder)
@@ -122,7 +106,7 @@ def _build(self, build_type=None, target=None):
         if target is not None:
             args = ["--target", target]
 
-        cmd_line_args = _cmake_cmd_line_args(self._conanfile, self._generator, self._parallel)
+        cmd_line_args = _cmake_cmd_line_args(self._conanfile, self._generator)
         if cmd_line_args:
             args += ['--'] + cmd_line_args
 
@@ -133,13 +117,9 @@ def _build(self, build_type=None, target=None):
         self._conanfile.run(command)
 
     def build(self, build_type=None, target=None):
-        if not self._conanfile.should_build:
-            return
         self._build(build_type, target)
 
     def install(self, build_type=None):
-        if not self._conanfile.should_install:
-            return
         mkdir(self._conanfile, self._conanfile.package_folder)
 
         bt = build_type or self._conanfile.settings.get_safe("build_type")
@@ -157,16 +137,10 @@ def install(self, build_type=None):
         self._conanfile.run(command)
 
     def test(self, build_type=None, target=None, output_on_failure=False):
-        if not self._conanfile.should_test:
-            return
         if self._conanfile.conf["tools.build:skip_test"]:
             return
         if not target:
             is_multi = is_multi_configuration(self._generator)
             target = "RUN_TESTS" if is_multi else "test"
 
-        env = {'CTEST_OUTPUT_ON_FAILURE': '1' if output_on_failure else '0'}
-        if self._parallel:
-            env['CTEST_PARALLEL_LEVEL'] = str(cpu_count(self._conanfile.output))
-        with tools.environment_append(env):
-            self._build(build_type=build_type, target=target)
+        self._build(build_type=build_type, target=target)
diff --git a/conan/tools/cmake/toolchain.py b/conan/tools/cmake/toolchain.py
@@ -8,6 +8,7 @@
 
 from conan.tools._check_build_profile import check_using_build_profile
 from conan.tools._compilers import architecture_flag, use_win_mingw
+from conan.tools.build import build_jobs
 from conan.tools.cmake.utils import is_multi_configuration
 from conan.tools.files import save_toolchain_args
 from conan.tools.intel import IntelCC
@@ -228,7 +229,7 @@ def context(self):
 
 class CppStdBlock(Block):
     template = textwrap.dedent("""
-        message(STATUS "Conan C++ Standard {{ cppstd }} with extensions {{ cppstd_extensions }}}")
+        message(STATUS "Conan toolchain: C++ Standard {{ cppstd }} with extensions {{ cppstd_extensions }}}")
         set(CMAKE_CXX_STANDARD {{ cppstd }})
         set(CMAKE_CXX_EXTENSIONS {{ cppstd_extensions }})
         set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -250,7 +251,7 @@ def context(self):
 
 class SharedLibBock(Block):
     template = textwrap.dedent("""
-        message(STATUS "Conan toolchain: Setting BUILD_SHARED_LIBS= {{ shared_libs }}")
+        message(STATUS "Conan toolchain: Setting BUILD_SHARED_LIBS = {{ shared_libs }}")
         set(BUILD_SHARED_LIBS {{ shared_libs }})
         """)
 
@@ -270,10 +271,14 @@ class ParallelBlock(Block):
 
     def context(self):
         # TODO: Check this conf
-        max_cpu_count = self._conanfile.conf["tools.cmake.cmaketoolchain:msvc_parallel_compile"]
 
-        if max_cpu_count:
-            return {"parallel": max_cpu_count}
+        compiler = self._conanfile.settings.get_safe("compiler")
+        if compiler not in ("Visual Studio", "msvc") or "Visual" not in self._toolchain.generator:
+            return
+
+        jobs = build_jobs(self._conanfile)
+        if jobs:
+            return {"parallel": jobs}
 
 
 class AndroidSystemBlock(Block):
@@ -433,7 +438,7 @@ def context(self):
             cppinfo.aggregate_components()
             build_paths.extend([os.path.join(req.package_folder,
                                        p.replace('\\', '/').replace('$', '\\$').replace('"', '\\"'))
-                                 for p in cppinfo.builddirs])
+                                for p in cppinfo.builddirs])
 
         if self._toolchain.find_builddirs:
             build_paths = " ".join(['"{}"'.format(b.replace('\\', '/')
@@ -695,7 +700,9 @@ class CMakeToolchain(object):
         #   CMAKE_CXX_FLAGS. See https://github.com/android/ndk/issues/323
         include_guard()
 
-        message("Using Conan toolchain through ${CMAKE_TOOLCHAIN_FILE}.")
+        if(CMAKE_TOOLCHAIN_FILE)
+            message("Using Conan toolchain: ${CMAKE_TOOLCHAIN_FILE}.")
+        endif()
 
         {% for conan_block in conan_blocks %}
         {{ conan_block }}

diff --git a/conan/tools/gnu/autotools.py b/conan/tools/gnu/autotools.py
@@ -1,6 +1,7 @@
 import os
+
+from conan.tools.build import build_jobs
 from conan.tools.files import load_toolchain_args
-from conan.tools.gnu.make import make_jobs_cmd_line_arg
 from conan.tools.microsoft.subsystems import subsystem_path, deduce_subsystem
 from conans.client.build import join_arguments
 
@@ -20,10 +21,6 @@ def configure(self, build_script_folder=None):
         http://jingfenghanmax.blogspot.com.es/2010/09/configure-with-host-target-and-build.html
         https://gcc.gnu.org/onlinedocs/gccint/Configure-Terms.html
         """
-        # FIXME: Conan 2.0 Are we keeping the "should_XXX" properties???
-        if not self._conanfile.should_configure:
-            return
-
         source = self._conanfile.source_folder
         if build_script_folder:
             source = os.path.join(self._conanfile.source_folder, build_script_folder)
@@ -43,13 +40,13 @@ def make(self, target=None):
         str_args = self._make_args
         jobs = ""
         if "-j" not in str_args and "nmake" not in make_program.lower():
-            jobs = make_jobs_cmd_line_arg(self._conanfile) or ""
+            njobs = build_jobs(self._conanfile)
+            if njobs:
+                jobs = "-j{}".format(njobs)
         command = join_arguments([make_program, target, str_args, jobs])
         self._conanfile.run(command)
 
     def install(self):
-        if not self._conanfile.should_install:
-            return
         self.make(target="install")
 
     def _use_win_mingw(self):

diff --git a/conan/tools/gnu/make.py b/conan/tools/gnu/make.py
diff --git a/conan/tools/meson/meson.py b/conan/tools/meson/meson.py
@@ -1,16 +1,10 @@
 import os
 
+from conan.tools.build import build_jobs
 from conan.tools.cross_building import cross_building
 from conan.tools.meson import MesonToolchain
 
 
-def ninja_jobs_cmd_line_arg(conanfile):
-    njobs = conanfile.conf["tools.ninja:jobs"] or \
-            conanfile.conf["tools.build:processes"]
-    if njobs:
-        return "-j{}".format(njobs)
-
-
 class Meson(object):
     def __init__(self, conanfile, build_folder='build'):
         self._conanfile = conanfile
@@ -41,9 +35,9 @@ def configure(self, source_folder=None):
 
     def build(self, target=None):
         cmd = 'meson compile -C "{}"'.format(self._build_dir)
-        njobs = ninja_jobs_cmd_line_arg(self._conanfile)
+        njobs = build_jobs(self._conanfile)
         if njobs:
-            cmd += " {}".format(njobs)
+            cmd += " -j{}".format(njobs)
         if target:
             cmd += " {}".format(target)
         self._conanfile.output.info("Meson build cmd: {}".format(cmd))

diff --git a/conan/tools/microsoft/msbuild.py b/conan/tools/microsoft/msbuild.py
@@ -9,13 +9,6 @@ def msbuild_verbosity_cmd_line_arg(conanfile):
         return '/verbosity:{}'.format(verbosity)
 
 
-def msbuild_max_cpu_count_cmd_line_arg(conanfile):
-    max_cpu_count = conanfile.conf["tools.microsoft.msbuild:max_cpu_count"] or \
-                    conanfile.conf["tools.build:processes"]
-    if max_cpu_count:
-        return "/m:{}".format(max_cpu_count)
-
-
 def msbuild_arch(arch):
     return {'x86': 'x86',
             'x86_64': 'x64',
@@ -43,9 +36,9 @@ def command(self, sln):
         if verbosity:
             cmd += " {}".format(verbosity)
 
-        max_cpu_count = msbuild_max_cpu_count_cmd_line_arg(self._conanfile)
-        if max_cpu_count:
-            cmd += " {}".format(max_cpu_count)
+        maxcpucount = self._conanfile.conf["tools.microsoft.msbuild:max_cpu_count"]
+        if maxcpucount:
+            cmd += " /m:{}".format(maxcpucount)
 
         return cmd
 

diff --git a/conan/tools/microsoft/toolchain.py b/conan/tools/microsoft/toolchain.py
@@ -3,6 +3,7 @@
 from xml.dom import minidom
 
 from conan.tools._check_build_profile import check_using_build_profile
+from conan.tools.build import build_jobs
 from conan.tools.intel.intel_cc import IntelCC
 from conan.tools.microsoft.visual import VCVars
 from conans.errors import ConanException
@@ -129,7 +130,7 @@ def format_macro(key, value):
                      {};%(PreprocessorDefinitions)
                   </PreprocessorDefinitions>
                   <RuntimeLibrary>{}</RuntimeLibrary>
-                  <LanguageStandard>{}</LanguageStandard>{}
+                  <LanguageStandard>{}</LanguageStandard>{}{}
                 </ClCompile>
                 <ResourceCompile>
                   <PreprocessorDefinitions>
@@ -152,10 +153,17 @@ def format_macro(key, value):
         if compile_options is not None:
             compile_options = eval(compile_options)
             self.compile_options.update(compile_options)
+        parallel = ""
+        njobs = build_jobs(self._conanfile)
+        if njobs:
+            parallel = "".join(
+                ["\n      <MultiProcessorCompilation>True</MultiProcessorCompilation>",
+                 "\n      <ProcessorNumber>{}</ProcessorNumber>".format(njobs)])
         compile_options = "".join("\n      <{k}>{v}</{k}>".format(k=k, v=v)
                                   for k, v in self.compile_options.items())
         config_props = toolchain_file.format(preprocessor_definitions, runtime_library, cppstd,
-                                             compile_options, preprocessor_definitions, toolset)
+                                             parallel, compile_options, preprocessor_definitions,
+                                             toolset)
         config_filepath = os.path.join(self._conanfile.generators_folder, config_filename)
         self._conanfile.output.info("MSBuildToolchain created %s" % config_filename)
         save(config_filepath, config_props)

diff --git a/conans/model/conf.py b/conans/model/conf.py
@@ -10,9 +10,8 @@
     "core:default_build_profile": "Defines the default build profile (None by default)",
     "tools.android:ndk_path": "Argument for the CMAKE_ANDROID_NDK",
     "tools.build:skip_test": "Do not execute CMake.test() and Meson.test() when enabled",
-    "tools.build:processes": "Default jobs number",
+    "tools.build:jobs": "Default compile jobs number -jX Ninja, Make, /MP VS (default: max CPUs)",
     "tools.cmake.cmaketoolchain:generator": "User defined CMake generator to use instead of default",
-    "tools.cmake.cmaketoolchain:msvc_parallel_compile": "Argument for the /MP when running msvc",
     "tools.cmake.cmaketoolchain:find_package_prefer_config": "Argument for the CMAKE_FIND_PACKAGE_PREFER_CONFIG",
     "tools.cmake.cmaketoolchain:toolchain_file": "Use other existing file rather than conan_toolchain.cmake one",
     "tools.cmake.cmaketoolchain:user_toolchain": "Inject existing user toolchain at the beginning of conan_toolchain.cmake",
@@ -23,17 +22,15 @@
     "tools.files.download:retry": "Number of retries in case of failure when downloading",
     "tools.files.download:retry_wait": "Seconds to wait between download attempts",
     "tools.gnu:make_program": "Indicate path to make program",
-    "tools.gnu.make:jobs": "Argument for the -j parameter when running Make generator",
     "tools.google.bazel:config": "Define Bazel config file",
     "tools.google.bazel:bazelrc_path": "Defines Bazel rc-path",
     "tools.microsoft.msbuild:verbosity": "Verbosity level for MSBuild: "
                                          "'Quiet', 'Minimal', 'Normal', 'Detailed', 'Diagnostic'",
-    "tools.microsoft.msbuild:max_cpu_count": "Argument for the /m (/maxCpuCount) when running MSBuild",
     "tools.microsoft.msbuild:vs_version": "Defines the IDE version when using the new msvc compiler",
+    "tools.microsoft.msbuild:max_cpu_count": "Argument for the /m when running msvc to build parallel projects",
     "tools.microsoft.msbuild:installation_path": "VS install path, to avoid auto-detect via vswhere, like C:/Program Files (x86)/Microsoft Visual Studio/2019/Community",
     "tools.microsoft.msbuilddeps:exclude_code_analysis": "Suppress MSBuild code analysis for patterns",
     "tools.microsoft.msbuildtoolchain:compile_options": "Dictionary with MSBuild compiler options",
-    "tools.ninja:jobs": "Argument for the --jobs parameter when running Ninja generator",
     "tools.intel:installation_path": "Defines the Intel oneAPI installation root path",
     "tools.intel:setvars_args": "Custom arguments to be passed onto the setvars.sh|bat script from Intel oneAPI"
 }

diff --git a/conans/test/functional/layout/test_editable_cmake.py b/conans/test/functional/layout/test_editable_cmake.py
@@ -6,7 +6,6 @@
 from conan.tools.env.environment import environment_wrap_command
 from conans.test.assets.pkg_cmake import pkg_cmake, pkg_cmake_app
 from conans.test.assets.sources import gen_function_cpp
-from conans.test.utils.mocks import ConanFileMock
 from conans.test.utils.tools import TestClient