diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 00e2cdd56ac..8a33254edf7 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -1265,6 +1265,7 @@ def get_graph_info(profile_names, settings, options, env, cwd, install_folder, c graph_info.root = root_ref lockfile = lockfile if os.path.isfile(lockfile) else os.path.join(lockfile, LOCKFILE) graph_lock_file = GraphLockFile.load(lockfile, cache.config.revisions_enabled) + graph_lock_file.graph_lock.relax = cache.config.relax_lockfile graph_info.profile_host = graph_lock_file.profile_host graph_info.profile_host.process_settings(cache, preprocess=False) graph_info.graph_lock = graph_lock_file.graph_lock diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index ee848fc2e21..86bfdb6ac30 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -468,6 +468,14 @@ def full_transitive_package_id(self): except ConanException: return None + @property + def relax_lockfile(self): + try: + fix_id = self.get_item("general.relax_lockfile") + return fix_id.lower() in ("1", "true") + except ConanException: + return None + @property def short_paths_home(self): short_paths_home = get_env("CONAN_USER_HOME_SHORT") diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index f4cd6be08d7..c06bb63ad69 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -1,7 +1,6 @@ import time -from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST, \ - CONTEXT_BUILD +from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod, conanfile_exception_formatter) from conans.model.conan_file import get_env_context_manager @@ -128,6 +127,8 @@ def _expand_node(self, node, graph, down_reqs, down_ref, down_options, check_upd def _resolve_ranges(self, graph, requires, consumer, update, remotes): for require in requires: + if require.locked_id: # if it is locked, nothing to resolved + continue self._resolver.resolve(require, consumer, update, remotes) self._resolve_cached_alias(requires, graph) @@ -153,27 +154,26 @@ def _get_node_requirements(self, node, graph, down_ref, down_options, down_reqs, if graph_lock: # No need to evaluate, they are hardcoded in lockfile graph_lock.lock_node(node, node.conanfile.requires.values()) - new_reqs = None - else: - # propagation of requirements only necessary if not locked - new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref) - # if there are version-ranges, resolve them before expanding each of the requirements - # Resolve possible version ranges of the current node requirements - # new_reqs is a shallow copy of what is propagated upstream, so changes done by the - # RangeResolver are also done in new_reqs, and then propagated! - conanfile = node.conanfile - scope = conanfile.display_name - self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes) - - if not hasattr(conanfile, "_conan_evaluated_requires"): - conanfile._conan_evaluated_requires = conanfile.requires.copy() - elif conanfile.requires != conanfile._conan_evaluated_requires: - raise ConanException("%s: Incompatible requirements obtained in different " - "evaluations of 'requirements'\n" - " Previous requirements: %s\n" - " New requirements: %s" - % (scope, list(conanfile._conan_evaluated_requires.values()), - list(conanfile.requires.values()))) + + # propagation of requirements can be necessary if some nodes are not locked + new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref) + # if there are version-ranges, resolve them before expanding each of the requirements + # Resolve possible version ranges of the current node requirements + # new_reqs is a shallow copy of what is propagated upstream, so changes done by the + # RangeResolver are also done in new_reqs, and then propagated! + conanfile = node.conanfile + scope = conanfile.display_name + self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes) + + if not hasattr(conanfile, "_conan_evaluated_requires"): + conanfile._conan_evaluated_requires = conanfile.requires.copy() + elif conanfile.requires != conanfile._conan_evaluated_requires: + raise ConanException("%s: Incompatible requirements obtained in different " + "evaluations of 'requirements'\n" + " Previous requirements: %s\n" + " New requirements: %s" + % (scope, list(conanfile._conan_evaluated_requires.values()), + list(conanfile.requires.values()))) return new_options, new_reqs @@ -272,10 +272,11 @@ def _conflicting_references(previous, new_ref, consumer_ref=None): if previous.ref.copy_clear_rev() != new_ref.copy_clear_rev(): if consumer_ref: return ("Conflict in %s:\n" - " '%s' requires '%s' while '%s' requires '%s'.\n" - " To fix this conflict you need to override the package '%s' in your root package." - % (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src, - previous.ref, new_ref.name)) + " '%s' requires '%s' while '%s' requires '%s'.\n" + " To fix this conflict you need to override the package '%s' " + "in your root package." + % (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src, + previous.ref, new_ref.name)) return True # Computed node, if is Editable, has revision=None # If new_ref.revision is None we cannot assume any conflict, the user hasn't specified @@ -316,7 +317,8 @@ def _config_node(node, down_ref, down_options): # Avoid extra time manipulating the sys.path for python with get_env_context_manager(conanfile, without_python=True): if hasattr(conanfile, "config"): - conan_v2_behavior("config() has been deprecated. Use config_options() and configure()", + conan_v2_behavior("config() has been deprecated. " + "Use config_options() and configure()", v1_behavior=conanfile.output.warn) with conanfile_exception_formatter(str(conanfile), "config"): conanfile.config() diff --git a/conans/model/graph_lock.py b/conans/model/graph_lock.py index b57e4ebb756..da39b4a672b 100644 --- a/conans/model/graph_lock.py +++ b/conans/model/graph_lock.py @@ -131,6 +131,7 @@ class GraphLock(object): def __init__(self, graph=None): self._nodes = {} # {numeric id: PREF or None} self.revisions_enabled = None + self.relax = False if graph is not None: for node in graph.nodes: @@ -164,7 +165,7 @@ def _upsert_node(self, node): python_reqs = node.conanfile.python_requires.all_refs() previous = node.graph_lock_node - modified = node.graph_lock_node.modified if node.graph_lock_node else None + modified = previous.modified if previous else None graph_node = GraphLockNode(node.pref if node.ref else None, python_reqs, node.conanfile.options.values, requires, build_requires, node.path, modified) @@ -274,7 +275,12 @@ def update_check_graph(self, deps_graph, output): except KeyError: if node.recipe == RECIPE_CONSUMER: continue # If the consumer node is not found, could be a test_package - raise + if self.relax: + continue + else: + raise ConanException("The node %s ID %s was not found in the lock" + % (node.ref, node.id)) + if lock_node.pref: pref = lock_node.pref if self.revisions_enabled else lock_node.pref.copy_clear_revs() node_pref = node.pref if self.revisions_enabled else node.pref.copy_clear_revs() @@ -296,7 +302,12 @@ def pre_lock_node(self, node): except KeyError: # If the consumer node is not found, could be a test_package if node.recipe == RECIPE_CONSUMER: return - raise ConanException("The node ID %s was not found in the lock" % node.id) + if self.relax: + node.conanfile.output.warn("Package can't be locked, not found in the lockfile") + return + else: + raise ConanException("The node %s ID %s was not found in the lock" + % (node.ref, node.id)) node.graph_lock_node = locked_node node.conanfile.options.values = locked_node.options @@ -324,12 +335,12 @@ def lock_node(self, node, requires, build_requires=False): locked_pref, locked_id = prefs[require.ref.name] require.lock(locked_pref.ref, locked_id) except KeyError: - msg = "'%s' cannot be found in lockfile for this package\n" % require.ref.name - if build_requires: - msg += "Make sure it was locked with --build arguments while creating lockfile" + t = "Build-require" if build_requires else "Require" + msg = "%s '%s' cannot be found in lockfile" % (t, require.ref.name) + if self.relax: + node.conanfile.output.warn(msg) else: - msg += "If it is a new requirement, you need to create a new lockile" - raise ConanException(msg) + raise ConanException(msg) def python_requires(self, node_id): if self.revisions_enabled: diff --git a/conans/test/functional/graph_lock/dynamic_test.py b/conans/test/functional/graph_lock/dynamic_test.py new file mode 100644 index 00000000000..c58fd45a9ac --- /dev/null +++ b/conans/test/functional/graph_lock/dynamic_test.py @@ -0,0 +1,30 @@ +import json +import unittest + +from conans.test.utils.tools import TestClient, GenConanfile + + +class GraphLockDynamicTest(unittest.TestCase): + + def remove_dep_test(self): + # Removing a dependency do not modify the graph of the lockfile + client = TestClient() + client.save({"conanfile.py": GenConanfile()}) + client.run("create . LibA/0.1@") + client.run("create . LibB/0.1@") + client.save({"conanfile.py": GenConanfile().with_require_plain("LibA/0.1") + .with_require_plain("LibB/0.1")}) + client.run("create . LibC/0.1@") + client.save({"conanfile.py": GenConanfile().with_require_plain("LibC/0.1")}) + client.run("graph lock .") + lock1 = json.loads(client.load("conan.lock"))["graph_lock"]["nodes"] + self.assertEqual(4, len(lock1)) + self.assertIn("LibC", lock1["1"]["pref"]) + self.assertEqual(lock1["1"]["requires"], ["2", "3"]) + # Remove one dep in LibC + client.save({"conanfile.py": GenConanfile().with_require_plain("LibA/0.1")}) + client.run("create . LibC/0.1@ --lockfile") + lock2 = json.loads(client.load("conan.lock"))["graph_lock"]["nodes"] + self.assertEqual(4, len(lock2)) + self.assertIn("LibC", lock2["1"]["pref"]) + self.assertEqual(lock2["1"]["requires"], ["2", "3"]) diff --git a/conans/test/functional/graph_lock/graph_lock_test.py b/conans/test/functional/graph_lock/graph_lock_test.py index 4a86dba5807..409f903a300 100644 --- a/conans/test/functional/graph_lock/graph_lock_test.py +++ b/conans/test/functional/graph_lock/graph_lock_test.py @@ -684,8 +684,9 @@ def test_override(self): class GraphLockBuildRequireErrorTestCase(unittest.TestCase): - def test(self): + def test_not_locked_build_requires(self): # https://github.com/conan-io/conan/issues/5807 + # even if the build requires are not locked, the graph can be augmented to add them client = TestClient() client.save({"zlib.py": GenConanfile(), "harfbuzz.py": GenConanfile().with_require_plain("fontconfig/1.0"), @@ -718,16 +719,89 @@ def test(self): self.assertEqual(harf, nodes["3"]["pref"]) self.assertEqual(zlib, nodes["4"]["pref"]) - # Using the graphlock there is no warning message - client.run("graph build-order . --build cascade --build outdated", assert_error=True) - self.assertIn("ERROR: 'fontconfig' cannot be found in lockfile for this package", client.out) - self.assertIn("Make sure it was locked ", client.out) + client.run("config set general.relax_lockfile=1") + client.run("graph build-order . --build cascade --build outdated --json=bo.json") + self.assertIn("ffmpeg/1.0: WARN: Build-require 'fontconfig' cannot be found in lockfile", + client.out) + self.assertIn("ffmpeg/1.0: WARN: Build-require 'harfbuzz' cannot be found in lockfile", + client.out) + lock = json.loads(client.load("conan.lock")) + nodes = lock["graph_lock"]["nodes"] + self.assertEqual(5, len(nodes)) + self.assertEqual(fmpe, nodes["1"]["pref"]) + # The lockfile doesn't add build_requires + self.assertEqual(None, nodes["1"].get("build_requires")) + self.assertEqual(font, nodes["2"]["pref"]) + self.assertEqual(harf, nodes["3"]["pref"]) + self.assertEqual(zlib, nodes["4"]["pref"]) + + build_order = json.loads(client.load("bo.json")) + self.assertEqual([["5", font]], build_order[0]) + self.assertEqual([["6", harf]], build_order[1]) + self.assertEqual([["1", fmpe], ["4", zlib]], build_order[2]) + + def test_build_requires_should_be_locked(self): + # https://github.com/conan-io/conan/issues/5807 + # this is the recommended approach, build_requires should be locked from the beginning + client = TestClient() + client.save({"zlib.py": GenConanfile(), + "harfbuzz.py": GenConanfile().with_require_plain("fontconfig/1.0"), + "fontconfig.py": GenConanfile(), + "ffmpeg.py": GenConanfile().with_build_require_plain("fontconfig/1.0") + .with_build_require_plain("harfbuzz/1.0"), + "variant.py": GenConanfile().with_require_plain("ffmpeg/1.0") + .with_require_plain("fontconfig/1.0") + .with_require_plain("harfbuzz/1.0") + .with_require_plain("zlib/1.0") + }) + client.run("export zlib.py zlib/1.0@") + client.run("export fontconfig.py fontconfig/1.0@") + client.run("export harfbuzz.py harfbuzz/1.0@") + client.run("export ffmpeg.py ffmpeg/1.0@") + + # Building the graphlock we get the message + client.run("graph lock variant.py --build") + fmpe = "ffmpeg/1.0#5522e93e2abfbd455e6211fe4d0531a2:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9" + font = "fontconfig/1.0#f3367e0e7d170aa12abccb175fee5f97:"\ + "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9" + harf = "harfbuzz/1.0#3172f5e84120f235f75f8dd90fdef84f:"\ + "ea61889683885a5517800e8ebb09547d1d10447a" + zlib = "zlib/1.0#f3367e0e7d170aa12abccb175fee5f97:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9" + lock = json.loads(client.load("conan.lock")) + nodes = lock["graph_lock"]["nodes"] + self.assertEqual(7, len(nodes)) + self.assertEqual(fmpe, nodes["1"]["pref"]) + self.assertEqual(["5", "6"], nodes["1"]["build_requires"]) + self.assertEqual(font, nodes["2"]["pref"]) + self.assertEqual(harf, nodes["3"]["pref"]) + self.assertEqual(zlib, nodes["4"]["pref"]) + self.assertEqual(font, nodes["5"]["pref"]) + self.assertEqual(harf, nodes["6"]["pref"]) + + client.run("graph build-order . --build cascade --build outdated --json=bo.json") + self.assertNotIn("cannot be found in lockfile", client.out) + lock = json.loads(client.load("conan.lock")) + nodes = lock["graph_lock"]["nodes"] + self.assertEqual(7, len(nodes)) + self.assertEqual(fmpe, nodes["1"]["pref"]) + self.assertEqual(["5", "6"], nodes["1"]["build_requires"]) + self.assertEqual(font, nodes["2"]["pref"]) + self.assertEqual(harf, nodes["3"]["pref"]) + self.assertEqual(zlib, nodes["4"]["pref"]) + self.assertEqual(font, nodes["5"]["pref"]) + self.assertEqual(harf, nodes["6"]["pref"]) + + build_order = json.loads(client.load("bo.json")) + self.assertEqual([["5", font]], build_order[0]) + self.assertEqual([["6", harf]], build_order[1]) + self.assertEqual([["1", fmpe], ["4", zlib]], build_order[2]) class GraphLockModifyConanfileTestCase(unittest.TestCase): def test(self): # https://github.com/conan-io/conan/issues/5807 + # Modifying dependencies do NOT modify the lockfile client = TestClient() client.save({"conanfile.py": GenConanfile()}) client.run("create . zlib/1.0@") @@ -736,9 +810,15 @@ def test(self): client2.save({"conanfile.py": GenConanfile()}) client2.run("graph lock .") client2.save({"conanfile.py": GenConanfile().with_require_plain("zlib/1.0")}) - client2.run("install . --lockfile", assert_error=True) - self.assertIn("ERROR: 'zlib' cannot be found in lockfile for this package", client2.out) - self.assertIn("If it is a new requirement, you need to create a new lockile", client2.out) + + client2.run("config set general.relax_lockfile=1") + client2.run("install . --lockfile") + self.assertIn("conanfile.py: WARN: Require 'zlib' cannot be found in lockfile", client2.out) + self.assertIn("zlib/1.0: WARN: Package can't be locked", client2.out) + self.assertIn("zlib/1.0:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client2.out) + lock_file_json = json.loads(client2.load("conan.lock")) + self.assertNotIn("zlib", lock_file_json) + self.assertEqual(1, len(lock_file_json["graph_lock"]["nodes"])) class LockFileOptionsTest(unittest.TestCase): diff --git a/conans/test/functional/graph_lock/test_package_test.py b/conans/test/functional/graph_lock/test_package_test.py index cf9e70608a3..5b3fd293073 100644 --- a/conans/test/functional/graph_lock/test_package_test.py +++ b/conans/test/functional/graph_lock/test_package_test.py @@ -7,8 +7,6 @@ class GraphLockTestPackageTest(unittest.TestCase): def augment_test_package_requires(self): # https://github.com/conan-io/conan/issues/6067 - # At this moment, it is not possible to add new nodes to a locked graph, which means - # test_package with build_requires raise errors client = TestClient() client.save({"conanfile.py": GenConanfile().with_name("tool").with_version("0.1")}) client.run("create .") @@ -26,10 +24,10 @@ def test(self): client.run("export .") client.run("graph lock consumer.txt -pr=profile --build missing") - lock = client.load("conan.lock") - client.run("create . -pr=profile --lockfile --build missing", assert_error=True) - self.assertIn("ERROR: The node ID 5 was not found in the lock", client.out) - # This would be the expected succesful output - # self.assertIn("tool/0.1:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client.out) - # self.assertIn("dep/0.1: Applying build-requirement: tool/0.1", client.out) - # self.assertIn("dep/0.1 (test package): Running test()", client.out) + + # Check lock + client.run("config set general.relax_lockfile=1") + client.run("create . -pr=profile --lockfile --build missing") + self.assertIn("tool/0.1:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client.out) + self.assertIn("dep/0.1: Applying build-requirement: tool/0.1", client.out) + self.assertIn("dep/0.1 (test package): Running test()", client.out) diff --git a/conans/test/unittests/client/generators/text_test.py b/conans/test/unittests/client/generators/text_test.py index 8e0ffe727c6..f74c0bdf102 100644 --- a/conans/test/unittests/client/generators/text_test.py +++ b/conans/test/unittests/client/generators/text_test.py @@ -49,7 +49,7 @@ def test_load_sytem_libs(self): content = textwrap.dedent(""" [system_libs] main - + [system_libs_requirement] requirement