Feature/dynamic lockfiles (#6457)

* allow adding deps in lockfiles

* working in augment

* working

* first version passing tests

* fix tests and build_info

* separate build-requires, sorted nodes, better status

* fix broken tests

* test checking older versions of lockfiles

* update docstring

* merged develop

* dynamic lockfiles

* dynamic test

* working with remove_orphans, but complicated for build_requires

* removed remove_orphans logic

* test passing

* [refact] Run 'conanfile::build()' only once in the sources (#6602)

* run build and related hooks in one single point

* duplicate arg

* prefer calling

* need to change the tests, message changes

* rename to 'run_build_method'

* build_folder is already assigned to the conanfile

* removing dead file (#6615)

* fix issue parsing system_libs field (#6616)

* Set environment variables in conaninfo.txt when using export-pkg (#6607)

* set conaninfo environment variables

* add test

* fix indentation

* refactor and filter deps

* fix variable name

* fix format

* capture the expected exception (#6622)

* new test

* opt-in relax_lockfile

Co-authored-by: Javier G. Sogo <[email protected]>
Co-authored-by: Carlos Zoido <[email protected]>

conans/client/conan_api.py

@@ -1265,6 +1265,7 @@ def get_graph_info(profile_names, settings, options, env, cwd, install_folder, c
             graph_info.root = root_ref
         lockfile = lockfile if os.path.isfile(lockfile) else os.path.join(lockfile, LOCKFILE)
         graph_lock_file = GraphLockFile.load(lockfile, cache.config.revisions_enabled)
+        graph_lock_file.graph_lock.relax = cache.config.relax_lockfile
         graph_info.profile_host = graph_lock_file.profile_host
         graph_info.profile_host.process_settings(cache, preprocess=False)
         graph_info.graph_lock = graph_lock_file.graph_lock

conans/client/conf/__init__.py

@@ -468,6 +468,14 @@ def full_transitive_package_id(self):
         except ConanException:
             return None
 
+    @property
+    def relax_lockfile(self):
+        try:
+            fix_id = self.get_item("general.relax_lockfile")
+            return fix_id.lower() in ("1", "true")
+        except ConanException:
+            return None
+
     @property
     def short_paths_home(self):
         short_paths_home = get_env("CONAN_USER_HOME_SHORT")

conans/client/graph/graph_builder.py

@@ -1,7 +1,6 @@
 import time
 
-from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST, \
-    CONTEXT_BUILD
+from conans.client.graph.graph import DepsGraph, Node, RECIPE_EDITABLE, CONTEXT_HOST
 from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod,
                            conanfile_exception_formatter)
 from conans.model.conan_file import get_env_context_manager
@@ -128,6 +127,8 @@ def _expand_node(self, node, graph, down_reqs, down_ref, down_options, check_upd
 
     def _resolve_ranges(self, graph, requires, consumer, update, remotes):
         for require in requires:
+            if require.locked_id:  # if it is locked, nothing to resolved
+                continue
             self._resolver.resolve(require, consumer, update, remotes)
         self._resolve_cached_alias(requires, graph)
 
@@ -153,27 +154,26 @@ def _get_node_requirements(self, node, graph, down_ref, down_options, down_reqs,
 
         if graph_lock:  # No need to evaluate, they are hardcoded in lockfile
             graph_lock.lock_node(node, node.conanfile.requires.values())
-            new_reqs = None
-        else:
-            # propagation of requirements only necessary if not locked
-            new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref)
-            # if there are version-ranges, resolve them before expanding each of the requirements
-            # Resolve possible version ranges of the current node requirements
-            # new_reqs is a shallow copy of what is propagated upstream, so changes done by the
-            # RangeResolver are also done in new_reqs, and then propagated!
-            conanfile = node.conanfile
-            scope = conanfile.display_name
-            self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes)
-
-            if not hasattr(conanfile, "_conan_evaluated_requires"):
-                conanfile._conan_evaluated_requires = conanfile.requires.copy()
-            elif conanfile.requires != conanfile._conan_evaluated_requires:
-                raise ConanException("%s: Incompatible requirements obtained in different "
-                                     "evaluations of 'requirements'\n"
-                                     "    Previous requirements: %s\n"
-                                     "    New requirements: %s"
-                                     % (scope, list(conanfile._conan_evaluated_requires.values()),
-                                        list(conanfile.requires.values())))
+
+        # propagation of requirements can be necessary if some nodes are not locked
+        new_reqs = node.conanfile.requires.update(down_reqs, self._output, node.ref, down_ref)
+        # if there are version-ranges, resolve them before expanding each of the requirements
+        # Resolve possible version ranges of the current node requirements
+        # new_reqs is a shallow copy of what is propagated upstream, so changes done by the
+        # RangeResolver are also done in new_reqs, and then propagated!
+        conanfile = node.conanfile
+        scope = conanfile.display_name
+        self._resolve_ranges(graph, conanfile.requires.values(), scope, update, remotes)
+
+        if not hasattr(conanfile, "_conan_evaluated_requires"):
+            conanfile._conan_evaluated_requires = conanfile.requires.copy()
+        elif conanfile.requires != conanfile._conan_evaluated_requires:
+            raise ConanException("%s: Incompatible requirements obtained in different "
+                                 "evaluations of 'requirements'\n"
+                                 "    Previous requirements: %s\n"
+                                 "    New requirements: %s"
+                                 % (scope, list(conanfile._conan_evaluated_requires.values()),
+                                    list(conanfile.requires.values())))
 
         return new_options, new_reqs
 
@@ -272,10 +272,11 @@ def _conflicting_references(previous, new_ref, consumer_ref=None):
         if previous.ref.copy_clear_rev() != new_ref.copy_clear_rev():
             if consumer_ref:
                 return ("Conflict in %s:\n"
-                       "    '%s' requires '%s' while '%s' requires '%s'.\n"
-                       "    To fix this conflict you need to override the package '%s' in your root package."
-                       % (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src,
-                          previous.ref, new_ref.name))
+                        "    '%s' requires '%s' while '%s' requires '%s'.\n"
+                        "    To fix this conflict you need to override the package '%s' "
+                        "in your root package."
+                        % (consumer_ref, consumer_ref, new_ref, next(iter(previous.dependants)).src,
+                           previous.ref, new_ref.name))
             return True
         # Computed node, if is Editable, has revision=None
         # If new_ref.revision is None we cannot assume any conflict, the user hasn't specified
@@ -316,7 +317,8 @@ def _config_node(node, down_ref, down_options):
             # Avoid extra time manipulating the sys.path for python
             with get_env_context_manager(conanfile, without_python=True):
                 if hasattr(conanfile, "config"):
-                    conan_v2_behavior("config() has been deprecated. Use config_options() and configure()",
+                    conan_v2_behavior("config() has been deprecated. "
+                                      "Use config_options() and configure()",
                                       v1_behavior=conanfile.output.warn)
                     with conanfile_exception_formatter(str(conanfile), "config"):
                         conanfile.config()

conans/model/graph_lock.py

@@ -131,6 +131,7 @@ class GraphLock(object):
     def __init__(self, graph=None):
         self._nodes = {}  # {numeric id: PREF or None}
         self.revisions_enabled = None
+        self.relax = False
 
         if graph is not None:
             for node in graph.nodes:
@@ -164,7 +165,7 @@ def _upsert_node(self, node):
             python_reqs = node.conanfile.python_requires.all_refs()
 
         previous = node.graph_lock_node
-        modified = node.graph_lock_node.modified if node.graph_lock_node else None
+        modified = previous.modified if previous else None
         graph_node = GraphLockNode(node.pref if node.ref else None, python_reqs,
                                    node.conanfile.options.values, requires, build_requires,
                                    node.path, modified)
@@ -274,7 +275,12 @@ def update_check_graph(self, deps_graph, output):
             except KeyError:
                 if node.recipe == RECIPE_CONSUMER:
                     continue  # If the consumer node is not found, could be a test_package
-                raise
+                if self.relax:
+                    continue
+                else:
+                    raise ConanException("The node %s ID %s was not found in the lock"
+                                         % (node.ref, node.id))
+
             if lock_node.pref:
                 pref = lock_node.pref if self.revisions_enabled else lock_node.pref.copy_clear_revs()
                 node_pref = node.pref if self.revisions_enabled else node.pref.copy_clear_revs()
@@ -296,7 +302,12 @@ def pre_lock_node(self, node):
         except KeyError:  # If the consumer node is not found, could be a test_package
             if node.recipe == RECIPE_CONSUMER:
                 return
-            raise ConanException("The node ID %s was not found in the lock" % node.id)
+            if self.relax:
+                node.conanfile.output.warn("Package can't be locked, not found in the lockfile")
+                return
+            else:
+                raise ConanException("The node %s ID %s was not found in the lock"
+                                     % (node.ref, node.id))
 
         node.graph_lock_node = locked_node
         node.conanfile.options.values = locked_node.options
@@ -324,12 +335,12 @@ def lock_node(self, node, requires, build_requires=False):
                 locked_pref, locked_id = prefs[require.ref.name]
                 require.lock(locked_pref.ref, locked_id)
             except KeyError:
-                msg = "'%s' cannot be found in lockfile for this package\n" % require.ref.name
-                if build_requires:
-                    msg += "Make sure it was locked with --build arguments while creating lockfile"
+                t = "Build-require" if build_requires else "Require"
+                msg = "%s '%s' cannot be found in lockfile" % (t, require.ref.name)
+                if self.relax:
+                    node.conanfile.output.warn(msg)
                 else:
-                    msg += "If it is a new requirement, you need to create a new lockile"
-                raise ConanException(msg)
+                    raise ConanException(msg)
 
     def python_requires(self, node_id):
         if self.revisions_enabled:

conans/test/functional/graph_lock/dynamic_test.py

@@ -0,0 +1,30 @@
+import json
+import unittest
+
+from conans.test.utils.tools import TestClient, GenConanfile
+
+
+class GraphLockDynamicTest(unittest.TestCase):
+
+    def remove_dep_test(self):
+        # Removing a dependency do not modify the graph of the lockfile
+        client = TestClient()
+        client.save({"conanfile.py": GenConanfile()})
+        client.run("create . LibA/0.1@")
+        client.run("create . LibB/0.1@")
+        client.save({"conanfile.py": GenConanfile().with_require_plain("LibA/0.1")
+                                                   .with_require_plain("LibB/0.1")})
+        client.run("create . LibC/0.1@")
+        client.save({"conanfile.py": GenConanfile().with_require_plain("LibC/0.1")})
+        client.run("graph lock .")
+        lock1 = json.loads(client.load("conan.lock"))["graph_lock"]["nodes"]
+        self.assertEqual(4, len(lock1))
+        self.assertIn("LibC", lock1["1"]["pref"])
+        self.assertEqual(lock1["1"]["requires"], ["2", "3"])
+        # Remove one dep in LibC
+        client.save({"conanfile.py": GenConanfile().with_require_plain("LibA/0.1")})
+        client.run("create . LibC/0.1@ --lockfile")
+        lock2 = json.loads(client.load("conan.lock"))["graph_lock"]["nodes"]
+        self.assertEqual(4, len(lock2))
+        self.assertIn("LibC", lock2["1"]["pref"])
+        self.assertEqual(lock2["1"]["requires"], ["2", "3"])

conans/test/functional/graph_lock/graph_lock_test.py

@@ -684,8 +684,9 @@ def test_override(self):
 
 class GraphLockBuildRequireErrorTestCase(unittest.TestCase):
 
-    def test(self):
+    def test_not_locked_build_requires(self):
         # https://github.com/conan-io/conan/issues/5807
+        # even if the build requires are not locked, the graph can be augmented to add them
         client = TestClient()
         client.save({"zlib.py": GenConanfile(),
                      "harfbuzz.py": GenConanfile().with_require_plain("fontconfig/1.0"),
@@ -718,16 +719,89 @@ def test(self):
         self.assertEqual(harf, nodes["3"]["pref"])
         self.assertEqual(zlib, nodes["4"]["pref"])
 
-        # Using the graphlock there is no warning message
-        client.run("graph build-order . --build cascade --build outdated", assert_error=True)
-        self.assertIn("ERROR: 'fontconfig' cannot be found in lockfile for this package", client.out)
-        self.assertIn("Make sure it was locked ", client.out)
+        client.run("config set general.relax_lockfile=1")
+        client.run("graph build-order . --build cascade --build outdated --json=bo.json")
+        self.assertIn("ffmpeg/1.0: WARN: Build-require 'fontconfig' cannot be found in lockfile",
+                      client.out)
+        self.assertIn("ffmpeg/1.0: WARN: Build-require 'harfbuzz' cannot be found in lockfile",
+                      client.out)
+        lock = json.loads(client.load("conan.lock"))
+        nodes = lock["graph_lock"]["nodes"]
+        self.assertEqual(5, len(nodes))
+        self.assertEqual(fmpe, nodes["1"]["pref"])
+        # The lockfile doesn't add build_requires
+        self.assertEqual(None, nodes["1"].get("build_requires"))
+        self.assertEqual(font, nodes["2"]["pref"])
+        self.assertEqual(harf, nodes["3"]["pref"])
+        self.assertEqual(zlib, nodes["4"]["pref"])
+
+        build_order = json.loads(client.load("bo.json"))
+        self.assertEqual([["5", font]], build_order[0])
+        self.assertEqual([["6", harf]], build_order[1])
+        self.assertEqual([["1", fmpe], ["4", zlib]], build_order[2])
+
+    def test_build_requires_should_be_locked(self):
+        # https://github.com/conan-io/conan/issues/5807
+        # this is the recommended approach, build_requires should be locked from the beginning
+        client = TestClient()
+        client.save({"zlib.py": GenConanfile(),
+                     "harfbuzz.py": GenConanfile().with_require_plain("fontconfig/1.0"),
+                     "fontconfig.py": GenConanfile(),
+                     "ffmpeg.py": GenConanfile().with_build_require_plain("fontconfig/1.0")
+                                                .with_build_require_plain("harfbuzz/1.0"),
+                     "variant.py": GenConanfile().with_require_plain("ffmpeg/1.0")
+                                                 .with_require_plain("fontconfig/1.0")
+                                                 .with_require_plain("harfbuzz/1.0")
+                                                 .with_require_plain("zlib/1.0")
+                     })
+        client.run("export zlib.py zlib/1.0@")
+        client.run("export fontconfig.py fontconfig/1.0@")
+        client.run("export harfbuzz.py harfbuzz/1.0@")
+        client.run("export ffmpeg.py ffmpeg/1.0@")
+
+        # Building the graphlock we get the message
+        client.run("graph lock variant.py --build")
+        fmpe = "ffmpeg/1.0#5522e93e2abfbd455e6211fe4d0531a2:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"
+        font = "fontconfig/1.0#f3367e0e7d170aa12abccb175fee5f97:"\
+               "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"
+        harf = "harfbuzz/1.0#3172f5e84120f235f75f8dd90fdef84f:"\
+               "ea61889683885a5517800e8ebb09547d1d10447a"
+        zlib = "zlib/1.0#f3367e0e7d170aa12abccb175fee5f97:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"
+        lock = json.loads(client.load("conan.lock"))
+        nodes = lock["graph_lock"]["nodes"]
+        self.assertEqual(7, len(nodes))
+        self.assertEqual(fmpe, nodes["1"]["pref"])
+        self.assertEqual(["5", "6"], nodes["1"]["build_requires"])
+        self.assertEqual(font, nodes["2"]["pref"])
+        self.assertEqual(harf, nodes["3"]["pref"])
+        self.assertEqual(zlib, nodes["4"]["pref"])
+        self.assertEqual(font, nodes["5"]["pref"])
+        self.assertEqual(harf, nodes["6"]["pref"])
+
+        client.run("graph build-order . --build cascade --build outdated --json=bo.json")
+        self.assertNotIn("cannot be found in lockfile", client.out)
+        lock = json.loads(client.load("conan.lock"))
+        nodes = lock["graph_lock"]["nodes"]
+        self.assertEqual(7, len(nodes))
+        self.assertEqual(fmpe, nodes["1"]["pref"])
+        self.assertEqual(["5", "6"], nodes["1"]["build_requires"])
+        self.assertEqual(font, nodes["2"]["pref"])
+        self.assertEqual(harf, nodes["3"]["pref"])
+        self.assertEqual(zlib, nodes["4"]["pref"])
+        self.assertEqual(font, nodes["5"]["pref"])
+        self.assertEqual(harf, nodes["6"]["pref"])
+
+        build_order = json.loads(client.load("bo.json"))
+        self.assertEqual([["5", font]], build_order[0])
+        self.assertEqual([["6", harf]], build_order[1])
+        self.assertEqual([["1", fmpe], ["4", zlib]], build_order[2])
 
 
 class GraphLockModifyConanfileTestCase(unittest.TestCase):
 
     def test(self):
         # https://github.com/conan-io/conan/issues/5807
+        # Modifying dependencies do NOT modify the lockfile
         client = TestClient()
         client.save({"conanfile.py": GenConanfile()})
         client.run("create . zlib/1.0@")
@@ -736,9 +810,15 @@ def test(self):
         client2.save({"conanfile.py": GenConanfile()})
         client2.run("graph lock .")
         client2.save({"conanfile.py": GenConanfile().with_require_plain("zlib/1.0")})
-        client2.run("install . --lockfile", assert_error=True)
-        self.assertIn("ERROR: 'zlib' cannot be found in lockfile for this package", client2.out)
-        self.assertIn("If it is a new requirement, you need to create a new lockile", client2.out)
+
+        client2.run("config set general.relax_lockfile=1")
+        client2.run("install . --lockfile")
+        self.assertIn("conanfile.py: WARN: Require 'zlib' cannot be found in lockfile", client2.out)
+        self.assertIn("zlib/1.0: WARN: Package can't be locked", client2.out)
+        self.assertIn("zlib/1.0:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client2.out)
+        lock_file_json = json.loads(client2.load("conan.lock"))
+        self.assertNotIn("zlib", lock_file_json)
+        self.assertEqual(1, len(lock_file_json["graph_lock"]["nodes"]))
 
 
 class LockFileOptionsTest(unittest.TestCase):

conans/test/functional/graph_lock/test_package_test.py

@@ -7,8 +7,6 @@
 class GraphLockTestPackageTest(unittest.TestCase):
     def augment_test_package_requires(self):
         # https://github.com/conan-io/conan/issues/6067
-        # At this moment, it is not possible to add new nodes to a locked graph, which means
-        # test_package with build_requires raise errors
         client = TestClient()
         client.save({"conanfile.py": GenConanfile().with_name("tool").with_version("0.1")})
         client.run("create .")
@@ -26,10 +24,10 @@ def test(self):
 
         client.run("export .")
         client.run("graph lock consumer.txt -pr=profile --build missing")
-        lock = client.load("conan.lock")
-        client.run("create . -pr=profile --lockfile --build missing", assert_error=True)
-        self.assertIn("ERROR: The node ID 5 was not found in the lock", client.out)
-        # This would be the expected succesful output
-        # self.assertIn("tool/0.1:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client.out)
-        # self.assertIn("dep/0.1: Applying build-requirement: tool/0.1", client.out)
-        # self.assertIn("dep/0.1 (test package): Running test()", client.out)
+
+        # Check lock
+        client.run("config set general.relax_lockfile=1")
+        client.run("create . -pr=profile --lockfile --build missing")
+        self.assertIn("tool/0.1:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache", client.out)
+        self.assertIn("dep/0.1: Applying build-requirement: tool/0.1", client.out)
+        self.assertIn("dep/0.1 (test package): Running test()", client.out)

conans/test/unittests/client/generators/text_test.py

@@ -49,7 +49,7 @@ def test_load_sytem_libs(self):
         content = textwrap.dedent("""
             [system_libs]
             main
-            
+
             [system_libs_requirement]
             requirement