From 6a7c1514f922f3a7d431b530de517a71bedd81a9 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 16 Nov 2022 12:34:22 +0100 Subject: [PATCH 01/62] improving README --- README.rst | 109 ++++++----------------------------------------------- test.sh | 0 tox.ini | 13 ------- 3 files changed, 11 insertions(+), 111 deletions(-) delete mode 100644 test.sh delete mode 100644 tox.ini diff --git a/README.rst b/README.rst index 91b3c398c23..a8ac30db90b 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ Decentralized, open-source (MIT), C/C++ package manager. - Homepage: https://conan.io/ - Github: https://github.com/conan-io/conan - Docs: https://docs.conan.io/en/latest/ -- Slack: https://cppalliance.org/slack/ (#conan channel) +- Slack: https://cpplang.slack.com/?id=conan (#conan channel) - Twitter: https://twitter.com/conan_io @@ -36,21 +36,12 @@ Conan is a package manager for C and C++ developers: +-------------------------+-------------------------+ +This is the **developer/maintainer** documentation. For user documentation, go to https://docs.conan.io + Setup ===== -Please read https://docs.conan.io/en/latest/installation.html to know how to -install and start using Conan. TL;DR: - -.. code-block:: - - $ pip install conan - - -Install a development version ------------------------------ - -You can run **Conan** client and server in Windows, MacOS, and Linux. +You can run Conan from source in Windows, MacOS, and Linux: - **Install pip following** `pip docs`_. @@ -60,7 +51,7 @@ You can run **Conan** client and server in Windows, MacOS, and Linux. $ git clone https://github.com/conan-io/conan.git conan-io - NOTE: repository directory name matters, some directories are known to be problematic to run tests (e.g. `conan`). `conan-io` directory name was tested and guaranteed to be working. + NOTE: repository directory name matters, some directories are known to be problematic to run tests (e.g. `conan`). `conan-io` directory name was tested and guaranteed to be working. - **Install in editable mode** @@ -78,42 +69,8 @@ You can run **Conan** client and server in Windows, MacOS, and Linux. Consumer commands install Installs the requirements specified in a recipe (conanfile.py or conanfile.txt). - config Manages Conan configuration. - get Gets a file or list a directory of a given reference or package. - info Gets information about the dependency graph of a recipe. - search Searches package recipes and binaries in the local cache or a remote. Unless a - remote is specified only the local cache is searched. - Creator commands - new Creates a new package recipe template with a 'conanfile.py' and optionally, - 'test_package' testing files. - create Builds a binary package for a recipe (conanfile.py). - upload Uploads a recipe and binary packages to a remote. - export Copies the recipe (conanfile.py & associated files) to your local cache. - export-pkg Exports a recipe, then creates a package from local source and build folders. - test Tests a package consuming it from a conanfile.py with a test() method. - Package development commands - source Calls your local conanfile.py 'source()' method. - build Calls your local conanfile.py 'build()' method. - package Calls your local conanfile.py 'package()' method. - editable Manages editable packages (packages that reside in the user workspace, but are - consumed as if they were in the cache). - workspace Manages a workspace (a set of packages consumed from the user workspace that - belongs to the same project). - Misc commands - profile Lists profiles in the '.conan/profiles' folder, or shows profile details. - remote Manages the remote list and the package recipes associated with a remote. - user Authenticates against a remote with user/pass, caching the auth token. - imports Calls your local conanfile.py or conanfile.txt 'imports' method. - copy Copies conan recipes and packages to another user/channel. - remove Removes packages or binaries matching pattern from local cache or remote. - alias Creates and exports an 'alias package recipe'. - download Downloads recipe and binaries to the local cache, without using settings. - inspect Displays conanfile attributes, like name, version, and options. Works locally, - in local cache and remote. - help Shows help for a specific command. - lock Generates and manipulates lock files. - frogarian Conan The Frogarian - + ... + Conan commands. Type "conan -h" for help Contributing to the project @@ -128,28 +85,10 @@ some advise on how to write tests for Conan. Running the tests ================= -Using tox ---------- - -.. code-block:: bash - - $ python -m tox - -It will install the needed requirements and launch `pytest` skipping some heavy and slow tests. -If you want to run the full test suite: - -.. code-block:: bash - - $ python -m tox -e full - -Without tox ------------ - **Install python requirements** .. code-block:: bash - $ python -m pip install -r conans/requirements.txt $ python -m pip install -r conans/requirements_server.txt $ python -m pip install -r conans/requirements_dev.txt @@ -168,44 +107,18 @@ On Windows it would be (while being in the Conan root directory): $ set PYTHONPATH=. -Ensure that your ``cmake`` has version 2.8 or later. You can see the -version with the following command: +Conan test suite defines and configure some required tools (CMake, Ninja, etc) in the +``conftest.py`` and allows to define a custom ``conftest_user.py``. +Some specific versions, like cmake>=3.15 are necessary. -.. code-block:: bash - - $ cmake --version - -The appropriate values of ``CONAN_COMPILER`` and ``CONAN_COMPILER_VERSION`` depend on your -operating system and your requirements. - -These should work for the GCC from ``build-essential`` on Ubuntu 14.04: -.. code-block:: bash - - $ export CONAN_COMPILER=gcc - $ export CONAN_COMPILER_VERSION=4.8 - -These should work for OS X: - -.. code-block:: bash - - $ export CONAN_COMPILER=clang - $ export CONAN_COMPILER_VERSION=3.5 - -You can run the actual tests like this: +You can run the tests like this: .. code-block:: bash $ python -m pytest . -There are a couple of test attributes defined, as ``slow`` that you can use -to filter the tests, and do not execute them: - -.. code-block:: bash - - $ python -m pytest . -m "not slow" - A few minutes later it should print ``OK``: .. code-block:: bash diff --git a/test.sh b/test.sh deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 460b296ff51..00000000000 --- a/tox.ini +++ /dev/null @@ -1,13 +0,0 @@ -[tox] - -[testenv] -setenv = - PYTHONPATH = {toxinidir} # Needed to avoid errors with test modules importing other test modules -deps = -rconans/requirements_dev.txt -commands = pytest -m "not slow and not tool_svn" [] # substitute with tox' positional arguments - -[testenv:full] -setenv = - PYTHONPATH = {toxinidir} # Needed to avoid errors with test modules importing other test modules -deps = {[testenv]deps} -commands = pytest [] # substitute with tox' positional arguments From 14a3fb325cc4ee74a7e6d32d3595cfa1bae28088 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 21 Dec 2022 08:49:18 +0100 Subject: [PATCH 02/62] conan 1.57.0-dev --- conans/__init__.py | 2 +- conans/client/migrations_settings.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index 64000f184ae..fda098f9d4c 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -20,4 +20,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, REVISIONS] # Server is always with revisions DEFAULT_REVISION_V1 = "0" -__version__ = '1.56.0-dev' +__version__ = '1.57.0-dev' diff --git a/conans/client/migrations_settings.py b/conans/client/migrations_settings.py index a38eb30e9b8..bdb5c3311bb 100644 --- a/conans/client/migrations_settings.py +++ b/conans/client/migrations_settings.py @@ -4185,3 +4185,5 @@ """ settings_1_56_0 = settings_1_55_0 + +settings_1_57_0 = settings_1_56_0 From 605ff02a42e2d3d540d4e79003887bee86e41ed4 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 23 Dec 2022 13:31:14 +0100 Subject: [PATCH 03/62] [develop2] remove unnecessary xz checks (#12777) --- conans/client/remote_manager.py | 14 ----- conans/client/rest/rest_client_v2.py | 6 +- conans/test/unittests/util/xz_test.py | 89 ++++----------------------- 3 files changed, 13 insertions(+), 96 deletions(-) diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 04f0b117fb6..4d527f1803b 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -76,7 +76,6 @@ def get_recipe(self, ref, remote): export_folder = layout.export() tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None) - check_compressed_files(EXPORT_TGZ_NAME, zipped_files) if tgz_file: uncompress_file(tgz_file, export_folder) mkdir(export_folder) @@ -101,7 +100,6 @@ def get_recipe_sources(self, ref, layout, remote): log_recipe_sources_download(ref, duration, remote.name, zipped_files) tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME] - check_compressed_files(EXPORT_SOURCES_TGZ_NAME, zipped_files) uncompress_file(tgz_file, export_sources_folder) def get_package(self, conanfile, pref, remote): @@ -129,7 +127,6 @@ def _get_package(self, layout, pref, remote, scoped_output): log_package_download(pref, duration, remote, zipped_files) tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None) - check_compressed_files(PACKAGE_TGZ_NAME, zipped_files) package_folder = layout.package() if tgz_file: # This must happen always, but just in case # TODO: The output could be changed to the package one, but @@ -213,17 +210,6 @@ def _call_remote(self, remote, method, *args, **kwargs): raise ConanException(exc, remote=remote) -# TODO: Consider removing this, we are not changing the compression format -def check_compressed_files(tgz_name, files): - bare_name = os.path.splitext(tgz_name)[0] - for f in files: - if f == tgz_name: - continue - if bare_name == os.path.splitext(f)[0]: - raise ConanException("This Conan version is not prepared to handle '%s' file format. " - "Please upgrade conan client." % f) - - def uncompress_file(src_path, dest_folder): t1 = time.time() try: diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index a1132e21cb6..de3367b637b 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -5,14 +5,13 @@ from conan.api.output import ConanOutput from conans.client.downloaders.caching_file_downloader import CachingFileDownloader -from conans.client.remote_manager import check_compressed_files from conans.client.rest.client_routes import ClientV2Router from conans.client.rest.file_uploader import FileUploader from conans.client.rest.rest_client_common import RestCommonMethods, get_exception_from_error from conans.errors import ConanException, NotFoundException, PackageNotFoundException, \ RecipeNotFoundException, AuthenticationException, ForbiddenException from conans.model.package_ref import PkgReference -from conans.paths import EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME, PACKAGE_TGZ_NAME +from conans.paths import EXPORT_SOURCES_TGZ_NAME from conans.util.dates import from_iso8601_to_timestamp @@ -47,7 +46,6 @@ def get_recipe(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] - check_compressed_files(EXPORT_TGZ_NAME, files) if EXPORT_SOURCES_TGZ_NAME in files: files.remove(EXPORT_SOURCES_TGZ_NAME) @@ -64,7 +62,6 @@ def get_recipe_sources(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) files = data["files"] - check_compressed_files(EXPORT_SOURCES_TGZ_NAME, files) if EXPORT_SOURCES_TGZ_NAME not in files: return None files = [EXPORT_SOURCES_TGZ_NAME, ] @@ -79,7 +76,6 @@ def get_package(self, pref, dest_folder): url = self.router.package_snapshot(pref) data = self._get_file_list_json(url) files = data["files"] - check_compressed_files(PACKAGE_TGZ_NAME, files) # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.package_file(pref, fn) for fn in files} self._download_and_save_files(urls, dest_folder, files) diff --git a/conans/test/unittests/util/xz_test.py b/conans/test/unittests/util/xz_test.py index 72b56a824b4..8945a245561 100644 --- a/conans/test/unittests/util/xz_test.py +++ b/conans/test/unittests/util/xz_test.py @@ -1,86 +1,21 @@ import os import tarfile -from unittest import TestCase from conan.tools.files import unzip -from conans.model.package_ref import PkgReference -from conans.model.recipe_ref import RecipeReference -from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.mocks import ConanFileMock from conans.test.utils.test_files import temp_folder -from conans.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, TestServer -from conans.util.files import load, save_files, save +from conans.util.files import load, save -class XZTest(TestCase): +def test_unzip_can_xz(): + tmp_dir = temp_folder() + file_path = os.path.join(tmp_dir, "a_file.txt") + save(file_path, "my content!") + txz = os.path.join(tmp_dir, "sample.tar.xz") + with tarfile.open(txz, "w:xz") as tar: + tar.add(file_path, "a_file.txt") - def test_error_xz(self): - server = TestServer() - ref = RecipeReference.loads("pkg/0.1@user/channel") - ref.revision = "myreciperev" - export = server.server_store.export(ref) - server.server_store.update_last_revision(ref) - save_files(export, {"conanfile.py": str(GenConanfile()), - "conanmanifest.txt": "#", - "conan_export.txz": "#"}) - client = TestClient(servers={"default": server}) - client.run("install --requires=pkg/0.1@user/channel", assert_error=True) - self.assertIn("This Conan version is not prepared to handle " - "'conan_export.txz' file format", client.out) - - def test_error_sources_xz(self): - server = TestServer() - ref = RecipeReference.loads("pkg/0.1@user/channel") - ref.revision = "myreciperev" - client = TestClient(servers={"default": server}) - server.server_store.update_last_revision(ref) - export = server.server_store.export(ref) - conanfile = """from conan import ConanFile -class Pkg(ConanFile): - exports_sources = "*" -""" - save_files(export, {"conanfile.py": conanfile, - "conanmanifest.txt": "1", - "conan_sources.txz": "#"}) - - client.run("install --requires=pkg/0.1@user/channel --build='*'", assert_error=True) - self.assertIn("ERROR: This Conan version is not prepared to handle " - "'conan_sources.txz' file format", client.out) - - def test_error_package_xz(self): - server = TestServer() - ref = RecipeReference.loads("pkg/0.1@user/channel") - ref.revision = "myreciperev" - client = TestClient(servers={"default": server}) - server.server_store.update_last_revision(ref) - export = server.server_store.export(ref) # *1 the path can't be known before upload a revision - conanfile = """from conan import ConanFile -class Pkg(ConanFile): - exports_sources = "*" -""" - save_files(export, {"conanfile.py": conanfile, - "conanmanifest.txt": "1"}) - pref = PkgReference(ref, NO_SETTINGS_PACKAGE_ID, "mypackagerev") - pref.revision = "mypackagerev" - server.server_store.update_last_package_revision(pref) - - package = server.server_store.package(pref) - save_files(package, {"conaninfo.txt": "#", - "conanmanifest.txt": "1", - "conan_package.txz": "#"}) - client.run("install --requires=pkg/0.1@user/channel", assert_error=True) - self.assertIn("ERROR: This Conan version is not prepared to handle " - "'conan_package.txz' file format", client.out) - - def test(self): - tmp_dir = temp_folder() - file_path = os.path.join(tmp_dir, "a_file.txt") - save(file_path, "my content!") - txz = os.path.join(tmp_dir, "sample.tar.xz") - with tarfile.open(txz, "w:xz") as tar: - tar.add(file_path, "a_file.txt") - - dest_folder = temp_folder() - unzip(ConanFileMock(), txz, dest_folder) - content = load(os.path.join(dest_folder, "a_file.txt")) - self.assertEqual(content, "my content!") + dest_folder = temp_folder() + unzip(ConanFileMock(), txz, dest_folder) + content = load(os.path.join(dest_folder, "a_file.txt")) + assert content == "my content!" From 3b2a26be49d09d2b4173247ff0e51819ddd26d9d Mon Sep 17 00:00:00 2001 From: James Date: Fri, 23 Dec 2022 13:32:43 +0100 Subject: [PATCH 04/62] [develop2] Test/develop2 false affect package (#12771) * [develop2] test that false values affect package_id * [develop2] test that false values affect package_id --- conans/test/unittests/model/info/__init__.py | 2 -- conans/test/unittests/model/info_test.py | 25 +++++++++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) delete mode 100644 conans/test/unittests/model/info/__init__.py diff --git a/conans/test/unittests/model/info/__init__.py b/conans/test/unittests/model/info/__init__.py deleted file mode 100644 index faa18be5bbf..00000000000 --- a/conans/test/unittests/model/info/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/conans/test/unittests/model/info_test.py b/conans/test/unittests/model/info_test.py index 110b4b44c3c..908e13226e9 100644 --- a/conans/test/unittests/model/info_test.py +++ b/conans/test/unittests/model/info_test.py @@ -2,7 +2,30 @@ import pytest -from conans.model.info import ConanInfo +from conans.model.info import ConanInfo, RequirementsInfo, PythonRequiresInfo +from conans.model.options import Options +from conans.model.settings import Settings + + +def test_false_values_affect_none(): + """ False value do become part of conaninfo.txt and package_id + Only "None" Python values are discarded from conaninfo.txt + """ + reqs = RequirementsInfo({}) + build_reqs = RequirementsInfo({}) + python_reqs = PythonRequiresInfo({}, default_package_id_mode=None) + options = Options({"shared": [True, False], "other": [None, "1"]}, {"shared": False}) + settings = Settings({"mysetting": [1, 2, 3]}) + c = ConanInfo(settings, options, reqs, build_reqs, python_reqs) + conaninfo = c.dumps() + assert "shared=False" in conaninfo + assert "other" not in conaninfo + assert "mysetting" not in conaninfo + + settings.mysetting = 1 + conaninfo = c.dumps() + assert "mysetting=1" in conaninfo + info_text = '''[settings] arch=x86_64 From c4b17a5a8517daa436cf19f32697fd992871fef7 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 27 Dec 2022 12:50:56 +0100 Subject: [PATCH 05/62] allow compatibility() for lockfiles (#12778) --- conans/client/graph/graph_binaries.py | 1 + .../graph_lock/test_compatibility.py | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 conans/test/integration/graph_lock/test_compatibility.py diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index e94c59978e9..a4c2dbb53ad 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -193,6 +193,7 @@ def _evaluate_node(self, node, build_mode, update, remotes): if node.package_id != locked.package_id: # It was a compatible package # https://github.com/conan-io/conan/issues/9002 # We need to iterate to search the compatible combination + self._compatibility.compatibles(node.conanfile) for compatible_package in node.conanfile.compatible_packages: comp_package_id = compatible_package.package_id() if comp_package_id == locked.package_id: diff --git a/conans/test/integration/graph_lock/test_compatibility.py b/conans/test/integration/graph_lock/test_compatibility.py new file mode 100644 index 00000000000..95271e00fd3 --- /dev/null +++ b/conans/test/integration/graph_lock/test_compatibility.py @@ -0,0 +1,38 @@ +import textwrap + +from conans.test.assets.genconanfile import GenConanfile +from conans.test.utils.tools import TestClient + + +def test_lockfile_compatibility(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "pkg" + version = "1.0" + settings = "build_type" + def compatibility(self): + if self.settings.build_type == "Release": + return [ {"settings": [("build_type", "None")]}, ] + """) + c.save({"conanfile.py": conanfile, + "profile": ""}) + c.run("create . -pr=profile") + c.save({"conanfile.py": GenConanfile().with_requires("pkg/1.0")}) + c.run("install .") + assert "pkg/1.0: Main binary package '4024617540c4f240a6a5e8911b0de9ef38a11a72' missing. " \ + "Using compatible package '5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9'" in c.out + + c.run("lock create conanfile.py --base") + assert "pkg/1.0: Main binary package '4024617540c4f240a6a5e8911b0de9ef38a11a72' missing. " \ + "Using compatible package '5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9'" in c.out + + c.run("lock create conanfile.py --lockfile=conan.lock") + conan_lock = c.load("conan.lock") + assert '"package_id": "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"' in conan_lock + assert "pkg/1.0: Main binary package '4024617540c4f240a6a5e8911b0de9ef38a11a72' missing. " \ + "Using compatible package '5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9'" in c.out + + c.run("install . --lockfile=conan.lock") + assert "pkg/1.0:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Cache" in c.out From 83fe3e7fb25cfb703bc531f2128dcf2559872758 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 27 Dec 2022 12:53:16 +0100 Subject: [PATCH 06/62] [develop2] Recover remote settings/options headers (#12772) --- conans/client/graph/graph_binaries.py | 3 +- conans/client/remote_manager.py | 15 +- .../remote/test_request_headers.py | 164 ++---------------- 3 files changed, 26 insertions(+), 156 deletions(-) diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index a8213c8f11e..6acb52f87d8 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -56,7 +56,8 @@ def _get_package_from_remotes(self, node): pref = node.pref for r in self._selected_remotes: try: - latest_pref = self._remote_manager.get_latest_package_reference(pref, r) + info = node.conanfile.info + latest_pref = self._remote_manager.get_latest_package_reference(pref, r, info) results.append({'pref': latest_pref, 'remote': r}) if len(results) > 0 and not self._update: break diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 4d527f1803b..14bbb1d2d1a 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -179,9 +179,20 @@ def get_latest_recipe_reference(self, ref, remote): assert ref.revision is None, "get_latest_recipe_reference of a reference with revision" return self._call_remote(remote, "get_latest_recipe_reference", ref) - def get_latest_package_reference(self, pref, remote) -> PkgReference: + def get_latest_package_reference(self, pref, remote, info=None) -> PkgReference: assert pref.revision is None, "get_latest_package_reference of a reference with revision" - return self._call_remote(remote, "get_latest_package_reference", pref, headers=None) + # These headers are useful to know what configurations are being requested in the server + headers = None + if info: + headers = {} + settings = [f'{k}={v}' for k, v in info.settings.items()] + if settings: + headers['Conan-PkgID-Settings'] = ';'.join(settings) + options = [f'{k}={v}' for k, v in info.options.serialize().items() + if k in ("shared", "fPIC", "header_only")] + if options: + headers['Conan-PkgID-Options'] = ';'.join(options) + return self._call_remote(remote, "get_latest_package_reference", pref, headers=headers) def get_recipe_revision_reference(self, ref, remote) -> bool: assert ref.revision is not None, "recipe_exists needs a revision" diff --git a/conans/test/integration/remote/test_request_headers.py b/conans/test/integration/remote/test_request_headers.py index da7db063094..3e084302190 100644 --- a/conans/test/integration/remote/test_request_headers.py +++ b/conans/test/integration/remote/test_request_headers.py @@ -1,163 +1,21 @@ -import textwrap -import unittest - -import pytest - from conans.test.assets.genconanfile import GenConanfile -from conans.test.utils.tools import TestClient, TestServer, TestRequester +from conans.test.utils.tools import TestClient, TestRequester class RequesterClass(TestRequester): - requests = None - - def __init__(self, *args, **kwargs): - self.requests = [] - super(RequesterClass, self).__init__(*args, **kwargs) def get(self, url, headers=None, **kwargs): - self.requests.append((url, headers)) + print(f"URL: {url}-HEADERS: {headers}") return super(RequesterClass, self).get(url, headers=headers, **kwargs) -class RequestHeadersTestCase(unittest.TestCase): - """ Conan adds a header with the settings used to compute the package ID """ - - profile = textwrap.dedent(""" - [settings] - os=Macos - arch=x86_64 - compiler=apple-clang - compiler.version=11.0 - compiler.libcxx=libc++ - build_type=Release - """) - - conanfile = GenConanfile().with_settings('os', 'arch', 'compiler') \ - .with_option('opt1', [True, False]) \ - .with_option('shared', [True, False]) \ - .with_default_option('opt1', True) \ - .with_default_option('shared', False) - - def setUp(self): - test_server = TestServer(users={"user": "mypass"}) - self.servers = {"default": test_server} - t = TestClient(servers=self.servers, inputs=["user", "mypass"]) - t.save({'conanfile.py': self.conanfile, - 'profile': self.profile}) - t.run('create file.py --name=name --version=version --user=user --channel=channel ' - '--profile:host=profile') - t.run('upload name/version@user/channel -r default') - - def _get_header(self, requester, header_name): - hits = sum([header_name in headers for _, headers in requester.requests]) - assert hits <= 2 - for url, headers in requester.requests: - if header_name in headers: - self.assertTrue(url.endswith('/latest'), msg=url) - return headers.get(header_name) - - def _assert_settings_headers(self, settings_header, compiler_version='11.0'): - # It takes only the values that are relevant to the recipe - self.assertListEqual( - sorted(['os', 'arch', 'compiler', 'compiler.version', 'compiler.libcxx']), - sorted([it.split('=', 1)[0] for it in settings_header.split(';')])) - self.assertIn('os=Macos', settings_header) - self.assertIn('arch=x86_64', settings_header) - self.assertIn('compiler=apple-clang', settings_header) - self.assertIn('compiler.libcxx=libc++', settings_header) - self.assertIn('compiler.version={}'.format(compiler_version), settings_header) - self.assertNotIn('build_type', settings_header) - - def _assert_options_headers(self, options_header, shared_value='False'): - self.assertListEqual(['shared'], [it.split('=', 1)[0] for it in options_header.split(';')]) - self.assertIn('shared={}'.format(shared_value), options_header) - - def _get_test_client(self): - t = TestClient(requester_class=RequesterClass, servers=self.servers, - inputs=["admin", "password"]) - return t - - @pytest.mark.xfail(reason="Requester not injected anymore. Revisit") - def test_install_recipe_mismatch(self): - - t = self._get_test_client() - t.save({'profile': self.profile}) - t.run('install --requires=failing/version@user/channel --profile=profile', assert_error=True) - self.assertFalse(any([CONAN_REQUEST_HEADER_SETTINGS in headers for _, headers in - t.api.http_requester.requests])) - self.assertFalse(any([CONAN_REQUEST_HEADER_OPTIONS in headers for _, headers in - t.api.http_requester.requests])) - - @pytest.mark.xfail(reason="Requester not injected anymore. Revisit") - @pytest.mark.xfail(reason="cache2.0 revisit") - def test_install_package_match(self): - t = self._get_test_client() - t.save({'profile': self.profile}) - - # Package match - t.run('install --requires=name/version@user/channel --profile=profile') - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers) - - # Package mismatch (settings) - t.run('install --requires=name/version@user/channel --profile=profile -s compiler.version=12.0', - assert_error=True) - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header, compiler_version='12.0') - - # Package mismatch (options) - t.run('install--requires=name/version@user/channel --profile=profile -o shared=True', - assert_error=True) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers, shared_value='True') - - @pytest.mark.xfail(reason="Requester not injected anymore. Revisit") - @pytest.mark.xfail(reason="cache2.0 revisit") - def test_info_package_match(self): - t = self._get_test_client() - t.save({'profile': self.profile}) - - # Package match - t.run('info name/version@user/channel --profile=profile') - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers) - - # Package mismatch (settings) - t.run('info name/version@user/channel --profile=profile -s compiler.version=12.0') - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header, compiler_version='12.0') - - # Package mismatch (options) - t.run('install --requires=name/version@user/channel --profile=profile -o shared=True', - assert_error=True) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers, shared_value='True') - - @pytest.mark.xfail(reason="Requester not injected anymore. Revisit") - @pytest.mark.xfail(reason="cache2.0 revisit") - def test_install_as_requirement(self): - t = self._get_test_client() - t.save({'conanfile.py': GenConanfile().with_requires('name/version@user/channel'), - 'profile': self.profile}) - - # Requirement is found - t.run('install . consumer/version@ --profile=profile') - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers) - - # Requirement is not found (settings) - t.run('install . consumer/version@ --profile=profile -s compiler.version=12.0', - assert_error=True) - settings_header = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_SETTINGS) - self._assert_settings_headers(settings_header, compiler_version='12.0') +def test_request_info_headers(): + c = TestClient(requester_class=RequesterClass, default_server_user=True) + conanfile = GenConanfile("pkg", "0.1").with_settings('os', 'arch', 'compiler') \ + .with_shared_option(False) + c.save({'conanfile.py': conanfile}) + c.run("export .") + c.run("install --requires=pkg/0.1 -s arch=x86_64", assert_error=True) + assert "'Conan-PkgID-Options': 'shared=False'" in c.out + assert "'Conan-PkgID-Settings': 'arch=x86_64;" in c.out - # Requirement is not found (options) - t.run('install . consumer/version@ --profile=profile -o name/*:shared=True', assert_error=True) - options_headers = self._get_header(t.api.http_requester, CONAN_REQUEST_HEADER_OPTIONS) - self._assert_options_headers(options_headers, shared_value='True') From 57355891fbdfcb8c66f6fbcdcd4b0e417b3fed5a Mon Sep 17 00:00:00 2001 From: James Date: Tue, 27 Dec 2022 12:55:29 +0100 Subject: [PATCH 07/62] [develop2] minor improvements remote-update (#12770) --- conan/cli/commands/remote.py | 34 +++---------------- .../test/integration/command/remote_test.py | 9 +++-- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/conan/cli/commands/remote.py b/conan/cli/commands/remote.py index d2463da85ea..ff0cce86cc4 100644 --- a/conan/cli/commands/remote.py +++ b/conan/cli/commands/remote.py @@ -66,28 +66,16 @@ def remote_add(conan_api, parser, subparser, *args): """ subparser.add_argument("name", help="Name of the remote to add") subparser.add_argument("url", help="Url of the remote") - subparser.add_argument("--secure", dest="secure", action='store_true', - help="Don't allow insecure server connections when using SSL") subparser.add_argument("--insecure", dest="secure", action='store_false', help="Allow insecure server connections when using SSL") subparser.add_argument("--index", action=OnceArgument, type=int, help="Insert the remote at a specific position in the remote list") subparser.set_defaults(secure=True) args = parser.parse_args(*args) - index = _check_index_argument(args.index) r = Remote(args.name, args.url, args.secure, disabled=False) conan_api.remotes.add(r) - if index is not None: - conan_api.remotes.move(r, index) - - -def _check_index_argument(index): - if index is None: - return None - try: - return int(index) - except ValueError: - raise ConanException("index argument must be an integer") + if args.index is not None: + conan_api.remotes.move(r, args.index) @conan_subcommand() @@ -116,7 +104,7 @@ def remote_update(conan_api, parser, subparser, *args): help="Insert the remote at a specific position in the remote list") subparser.set_defaults(secure=None) args = parser.parse_args(*args) - if not (args.url is not None or args.secure is not None): + if args.url is None and args.secure is None and args.index is None: subparser.error("Please add at least one argument to update") r = conan_api.remotes.get(args.remote) if args.url is not None: @@ -124,6 +112,8 @@ def remote_update(conan_api, parser, subparser, *args): if args.secure is not None: r.verify_ssl = args.secure conan_api.remotes.update(r) + if args.index is not None: + conan_api.remotes.move(r, args.index) @conan_subcommand() @@ -138,20 +128,6 @@ def remote_rename(conan_api, parser, subparser, *args): conan_api.remotes.rename(r, args.new_name) -@conan_subcommand() -def remote_move(conan_api, parser, subparser, *args): - """ - Move the position of the remote in the remotes list (first in the list: first called) - """ - subparser.add_argument("remote", help="Name of the remote to update") - subparser.add_argument("index", action=OnceArgument, type=int, - help="Insert remote at specific index") - args = parser.parse_args(*args) - r = conan_api.remotes.get(args.remote) - index = _check_index_argument(args.index) - conan_api.remotes.move(r, index) - - @conan_subcommand() def remote_enable(conan_api, parser, subparser, *args): """ diff --git a/conans/test/integration/command/remote_test.py b/conans/test/integration/command/remote_test.py index d5afcded58a..caefd5fb04b 100644 --- a/conans/test/integration/command/remote_test.py +++ b/conans/test/integration/command/remote_test.py @@ -125,8 +125,7 @@ def test_update_insert(self): client.run("remote add r2 https://r2") client.run("remote add r3 https://r3") - client.run("remote update r2 --url https://r2new") - client.run("remote move r2 0") + client.run("remote update r2 --url https://r2new --index=0") client.run("remote list") lines = str(client.out).splitlines() self.assertIn("r2: https://r2new", lines[0]) @@ -134,7 +133,7 @@ def test_update_insert(self): self.assertIn("r3: https://r3", lines[2]) client.run("remote update r2 --url https://r2new2") - client.run("remote move r2 2") + client.run("remote update r2 --index=2") client.run("remote list") lines = str(client.out).splitlines() self.assertIn("r1: https://r1", lines[0]) @@ -148,14 +147,14 @@ def test_update_insert_same_url(self): client.run("remote add r2 https://r2") client.run("remote add r3 https://r3") client.run("remote update r2 --url https://r2") - client.run("remote move r2 0") + client.run("remote update r2 --index 0") client.run("remote list") self.assertLess(str(client.out).find("r2"), str(client.out).find("r1")) self.assertLess(str(client.out).find("r1"), str(client.out).find("r3")) def test_verify_ssl(self): client = TestClient() - client.run("remote add my-remote http://someurl --secure") + client.run("remote add my-remote http://someurl") client.run("remote add my-remote2 http://someurl2 --insecure") client.run("remote add my-remote3 http://someurl3") From 1debf48c61b4a889051dc625233dd926acbc7571 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 27 Dec 2022 15:03:03 +0100 Subject: [PATCH 08/62] [develop2] Allow ``export-pkg`` work with ``tool_requires`` used in the ``package()`` method (#12769) * wip * fix test --- conan/api/subapi/export.py | 20 ++--- conan/cli/commands/export_pkg.py | 15 +++- .../integration/command/export_pkg_test.py | 84 +++++++++++++++++-- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/conan/api/subapi/export.py b/conan/api/subapi/export.py index b58f2bc11f6..fd6ace2c6b9 100644 --- a/conan/api/subapi/export.py +++ b/conan/api/subapi/export.py @@ -1,5 +1,3 @@ -import os - from conan.api.output import ConanOutput from conan.api.subapi import api_method from conan.internal.conan_app import ConanApp @@ -22,7 +20,7 @@ def export(self, path, name, version, user, channel, lockfile=None, remotes=None remotes=remotes) @api_method - def export_pkg(self, deps_graph, path): + def export_pkg(self, deps_graph, source_folder, output_folder): app = ConanApp(self.conan_api.cache_folder) cache, hook_manager = app.cache, app.hook_manager @@ -30,9 +28,10 @@ def export_pkg(self, deps_graph, path): # to be downloaded from remotes # passing here the create_reference=ref argument is useful so the recipe is in "develop", # because the "package()" method is in develop=True already - pkg_node = deps_graph.root ref = pkg_node.ref + out = ConanOutput(scope=pkg_node.conanfile.display_name) + out.info("Exporting binary from user folder to Conan cache") if pkg_node.binary == BINARY_INVALID: binary, reason = "Invalid", pkg_node.conanfile.info.invalid msg = "{}: Invalid ID: {}: {}".format(ref, binary, reason) @@ -41,17 +40,13 @@ def export_pkg(self, deps_graph, path): package_id = pkg_node.package_id assert package_id is not None - ConanOutput().info("Packaging to %s" % package_id) + out.info("Packaging to %s" % package_id) pref = PkgReference(ref, package_id) pkg_layout = cache.create_build_pkg_layout(pref) + conanfile.folders.set_base_folders(source_folder, output_folder) dest_package_folder = pkg_layout.package() - - conanfile_folder = os.path.dirname(path) - conanfile.folders.set_base_build(conanfile_folder) - conanfile.folders.set_base_source(conanfile_folder) conanfile.folders.set_base_package(dest_package_folder) - conanfile.folders.set_base_generators(conanfile_folder) with pkg_layout.set_dirty_context_manager(): prev = run_package_method(conanfile, package_id, hook_manager, ref) @@ -60,4 +55,7 @@ def export_pkg(self, deps_graph, path): pkg_layout.reference = pref cache.assign_prev(pkg_layout) # Make sure folder is updated - conanfile.folders.set_base_package(pkg_layout.package()) + final_folder = pkg_layout.package() + conanfile.folders.set_base_package(final_folder) + out.info(f"Package folder {final_folder}") + out.success("Exported package binary") diff --git a/conan/cli/commands/export_pkg.py b/conan/cli/commands/export_pkg.py index 08a1f1dae2e..42720e9a5e8 100644 --- a/conan/cli/commands/export_pkg.py +++ b/conan/cli/commands/export_pkg.py @@ -4,6 +4,7 @@ from conan.api.output import cli_out_write from conan.cli.command import conan_command from conan.cli.args import add_lockfile_args, add_profiles_args, add_reference_args +from conan.cli.commands import make_abs_path from conans.errors import ConanInvalidConfiguration @@ -18,6 +19,9 @@ def export_pkg(conan_api, parser, *args): Export recipe to the Conan package cache, and create a package directly from pre-compiled binaries """ parser.add_argument("path", help="Path to a folder containing a recipe (conanfile.py)") + parser.add_argument("-of", "--output-folder", + help='The root output folder for generated and build files') + add_reference_args(parser) add_lockfile_args(parser) add_profiles_args(parser) @@ -29,7 +33,6 @@ def export_pkg(conan_api, parser, *args): conanfile_path=path, cwd=cwd, partial=args.lockfile_partial) - profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) ref, conanfile = conan_api.export.export(path=path, @@ -68,7 +71,15 @@ def export_pkg(conan_api, parser, *args): msg = "{}: {}: {}".format(conanfile, binary, reason) raise ConanInvalidConfiguration(msg) - conan_api.export.export_pkg(deps_graph, path) + # It is necessary to install binaries, in case there are build_requires necessary to export + # But they should be local, if this was built here + conan_api.install.install_binaries(deps_graph=deps_graph, remotes=None, update=False) + source_folder = os.path.dirname(path) + output_folder = make_abs_path(args.output_folder, cwd) if args.output_folder else None + conan_api.install.install_consumer(deps_graph=deps_graph, source_folder=source_folder, + output_folder=output_folder) + + conan_api.export.export_pkg(deps_graph, source_folder, output_folder) lockfile = conan_api.lockfile.update_lockfile(lockfile, deps_graph, args.lockfile_packages, clean=args.lockfile_clean) diff --git a/conans/test/integration/command/export_pkg_test.py b/conans/test/integration/command/export_pkg_test.py index 9513c3ce3fd..8bf87257042 100644 --- a/conans/test/integration/command/export_pkg_test.py +++ b/conans/test/integration/command/export_pkg_test.py @@ -529,7 +529,7 @@ def package_info(self): for n in nodes: ref = n["ref"] if ref == hello_pkg_ref: - assert n['binary'] == None # The exported package has no binary status + assert n['binary'] is None # The exported package has no binary status hello_cpp_info = n['cpp_info'] elif ref == pkg_pkg_ref: assert n['binary'] == "Cache" @@ -543,17 +543,19 @@ def package_info(self): assert hello_cpp_info['root']["properties"] is None # pkg/0.2 cpp_info # root info - assert pkg_cpp_info['root']["libs"] is None + + assert pkg_cpp_info['root']["libs"] == ['pkg'] assert len(pkg_cpp_info['root']["bindirs"]) == 1 assert len(pkg_cpp_info['root']["libdirs"]) == 1 - assert pkg_cpp_info['root']["sysroot"] is None - assert pkg_cpp_info['root']["system_libs"] is None - assert pkg_cpp_info['root']['cflags'] is None - assert pkg_cpp_info['root']['cxxflags'] is None - assert pkg_cpp_info['root']['defines'] is None - assert pkg_cpp_info['root']["properties"] is None + assert pkg_cpp_info['root']["sysroot"] == '/path/to/folder/pkg' + assert pkg_cpp_info['root']["system_libs"] == ['pkg_onesystemlib', 'pkg_twosystemlib'] + assert pkg_cpp_info['root']['cflags'] == ['pkg_a_c_flag'] + assert pkg_cpp_info['root']['cxxflags'] == ['pkg_a_cxx_flag'] + assert pkg_cpp_info['root']['defines'] == ['pkg_onedefinition', 'pkg_twodefinition'] + assert pkg_cpp_info['root']["properties"] == {'pkg_config_name': 'pkg_other_name', + 'pkg_config_aliases': ['pkg_alias1', 'pkg_alias2']} # component info - assert pkg_cpp_info.get('cmp1') is None + assert pkg_cpp_info["cmp1"]["libs"] == ["libcmp1"] def test_export_pkg_dont_update_src(): @@ -600,3 +602,67 @@ def test_negate_tool_requires(): "conanfile.py": GenConanfile("mypkg", "0.1")}) c.run("export-pkg . -pr=myprofile") assert "conanfile.py (mypkg/0.1): Created package" in c.out + + +def test_export_pkg_tool_requires(): + """ when a package has "tool_requires" that it needs at the package() method, like + typical cmake.install() or autotools.install() (tool_require msys2), then it is necessary: + - to install the dependencies + - to inject the tool-requirements + - to propagate the environment and the conf + """ + c = TestClient() + tool = textwrap.dedent(""" + from conan import ConanFile + class Tool(ConanFile): + name = "tool" + version = "0.1" + def package_info(self): + self.buildenv_info.define("MYVAR", "MYVALUE") + self.conf_info.define("user.team:conf", "CONF_VALUE") + """) + consumer = textwrap.dedent(""" + import platform + from conan import ConanFile + + class Consumer(ConanFile): + name = "consumer" + version = "0.1" + tool_requires = "tool/0.1" + def package(self): + self.output.info(f"MYCONF {self.conf.get('user.team:conf')}") + cmd = "set MYVAR" if platform.system() == "Windows" else "echo MYVAR=$MYVAR" + self.run(cmd) + """) + c.save({"tool/conanfile.py": tool, + "consumer/conanfile.py": consumer}) + + c.run("create tool") + c.run("export-pkg consumer") + assert "conanfile.py (consumer/0.1): MYCONF CONF_VALUE" in c.out + assert "MYVAR=MYVALUE" in c.out + + +def test_export_pkg_output_folder(): + """ If the local build is using a different output-folder, it should work and export it + """ + c = TestClient() + consumer = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import save, copy + + class Consumer(ConanFile): + name = "consumer" + version = "0.1" + + def build(self): + save(self, "myfile.txt", "") + def package(self): + copy(self, "*", src=self.build_folder, dst=self.package_folder) + """) + c.save({"conanfile.py": consumer}) + + c.run("build . -of=mytmp") + c.run("export-pkg . -of=mytmp") + assert "Copied 1 '.txt' file: myfile.txt" in c.out + assert os.path.exists(os.path.join(c.current_folder, "mytmp", "myfile.txt")) From 7b381791d3c207c0135c8030644f9f9e99e9ddc3 Mon Sep 17 00:00:00 2001 From: Josh Neu <36168237+neuj21@users.noreply.github.com> Date: Tue, 27 Dec 2022 08:06:42 -0600 Subject: [PATCH 09/62] Add cmake install component support #10359 (#12762) * fixes #10359: Add cmake install component support * Issue #10359: Fix unit test bug * Address comments from #12762 --- conan/tools/cmake/cmake.py | 4 +- .../tools/cmake/test_cmake_install.py | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 conans/test/unittests/tools/cmake/test_cmake_install.py diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py index 63cb6a1a1b7..9b8e1a0f0eb 100644 --- a/conan/tools/cmake/cmake.py +++ b/conan/tools/cmake/cmake.py @@ -132,7 +132,7 @@ def _build(self, build_type=None, target=None, cli_args=None, build_tool_args=No def build(self, build_type=None, target=None, cli_args=None, build_tool_args=None): self._build(build_type, target, cli_args, build_tool_args) - def install(self, build_type=None): + def install(self, build_type=None, component=None): mkdir(self._conanfile, self._conanfile.package_folder) bt = build_type or self._conanfile.settings.get_safe("build_type") @@ -144,6 +144,8 @@ def install(self, build_type=None): pkg_folder = '"{}"'.format(self._conanfile.package_folder.replace("\\", "/")) build_folder = '"{}"'.format(self._conanfile.build_folder) arg_list = ["--install", build_folder, build_config, "--prefix", pkg_folder] + if component: + arg_list.extend(["--component", component]) arg_list = " ".join(filter(None, arg_list)) command = "%s %s" % (self._cmake_program, arg_list) self._conanfile.output.info("CMake command: %s" % command) diff --git a/conans/test/unittests/tools/cmake/test_cmake_install.py b/conans/test/unittests/tools/cmake/test_cmake_install.py new file mode 100644 index 00000000000..273c388355b --- /dev/null +++ b/conans/test/unittests/tools/cmake/test_cmake_install.py @@ -0,0 +1,39 @@ +import platform +import pytest + +from conan.tools.cmake import CMake +from conan.tools.cmake.presets import write_cmake_presets +from conans.client.conf import get_default_settings_yml +from conans.model.conf import Conf +from conans.model.settings import Settings +from conans.test.utils.mocks import ConanFileMock +from conans.test.utils.test_files import temp_folder + + +def test_run_install_component(): + """ + Testing that the proper component is installed. + Issue related: https://github.com/conan-io/conan/issues/10359 + """ + # Load some generic windows settings + settings = Settings.loads(get_default_settings_yml()) + settings.os = "Windows" + settings.arch = "x86" + settings.build_type = "Release" + settings.compiler = "Visual Studio" + settings.compiler.runtime = "MDd" + settings.compiler.version = "14" + + conanfile = ConanFileMock() + conanfile.conf = Conf() + conanfile.folders.generators = "." + conanfile.folders.set_base_generators(temp_folder()) + conanfile.settings = settings + conanfile.folders.set_base_package(temp_folder()) + + # Choose generator to match generic setttings + write_cmake_presets(conanfile, "toolchain", "Visual Studio 14 2015", {}) + cmake = CMake(conanfile) + cmake.install(component="foo") + + assert "--component foo" in conanfile.command From ce9c7826d5aa18b2085ca28b40de4da4724b726a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Tue, 27 Dec 2022 15:07:17 +0100 Subject: [PATCH 10/62] Make ConanOutput write methods chainable (#12780) * Make ConanOutput write methods chainable * Add chaining to another test --- conan/api/output.py | 16 +++++++++++++--- .../py_requires/python_requires_test.py | 16 ++++++++-------- .../test/unittests/client/conan_output_test.py | 10 ++++++++++ 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/conan/api/output.py b/conan/api/output.py index 8a5c95ea96a..79b07bab18e 100644 --- a/conan/api/output.py +++ b/conan/api/output.py @@ -82,11 +82,11 @@ def is_terminal(self): return hasattr(self.stream, "isatty") and self.stream.isatty() def writeln(self, data, fg=None, bg=None): - self.write(data, fg, bg, newline=True) + return self.write(data, fg, bg, newline=True) def write(self, data, fg=None, bg=None, newline=False): if conan_output_level > LEVEL_NOTICE: - return + return self if self._color and (fg or bg): data = "%s%s%s%s" % (fg or '', bg or '', data, Style.RESET_ALL) @@ -94,12 +94,13 @@ def write(self, data, fg=None, bg=None, newline=False): data = "%s\n" % data self.stream.write(data) self.stream.flush() + return self def rewrite_line(self, line): tmp_color = self._color self._color = False TOTAL_SIZE = 70 - LIMIT_SIZE = 32 # Hard coded instead of TOTAL_SIZE/2-3 that fails in Py3 float division + LIMIT_SIZE = TOTAL_SIZE // 2 - 3 if len(line) > TOTAL_SIZE: line = line[0:LIMIT_SIZE] + " ... " + line[-LIMIT_SIZE:] self.write("\r%s%s" % (line, " " * (TOTAL_SIZE - len(line)))) @@ -148,18 +149,22 @@ def json_encoder(_obj): def trace(self, msg): if log_level_allowed(LEVEL_TRACE): self._write_message(msg, "TRACE", fg=Color.BRIGHT_WHITE) + return self def debug(self, msg): if log_level_allowed(LEVEL_DEBUG): self._write_message(msg, "DEBUG") + return self def verbose(self, msg, fg=None, bg=None): if log_level_allowed(LEVEL_VERBOSE): self._write_message(msg, "VERBOSE", fg=fg, bg=bg) + return self def status(self, msg, fg=None, bg=None): if log_level_allowed(LEVEL_STATUS): self._write_message(msg, "STATUS", fg=fg, bg=bg) + return self # Remove in a later refactor of all the output.info calls info = status @@ -168,22 +173,27 @@ def title(self, msg): if log_level_allowed(LEVEL_NOTICE): self._write_message("\n-------- {} --------".format(msg), "NOTICE", fg=Color.BRIGHT_MAGENTA) + return self def highlight(self, msg): if log_level_allowed(LEVEL_NOTICE): self._write_message(msg, "NOTICE", fg=Color.BRIGHT_MAGENTA) + return self def success(self, msg): if log_level_allowed(LEVEL_NOTICE): self._write_message(msg, "NOTICE", fg=Color.BRIGHT_GREEN) + return self def warning(self, msg): if log_level_allowed(LEVEL_WARNING): self._write_message("WARN: {}".format(msg), "WARN", Color.YELLOW) + return self def error(self, msg): if log_level_allowed(LEVEL_ERROR): self._write_message("ERROR: {}".format(msg), "ERROR", Color.RED) + return self def flush(self): self.stream.flush() diff --git a/conans/test/integration/py_requires/python_requires_test.py b/conans/test/integration/py_requires/python_requires_test.py index a6ad75071fb..6863ff5814f 100644 --- a/conans/test/integration/py_requires/python_requires_test.py +++ b/conans/test/integration/py_requires/python_requires_test.py @@ -214,14 +214,14 @@ class Pkg(ConanFile): class MyConanfileBase(ConanFile): python_requires = "pkg1/1.0@user/channel", "pkg2/1.0@user/channel" def build(self): - self.output.info("PKG1 N: %s" % self.python_requires["pkg1"].conanfile.name) - self.output.info("PKG1 V: %s" % self.python_requires["pkg1"].conanfile.version) - self.output.info("PKG1 U: %s" % self.python_requires["pkg1"].conanfile.user) - self.output.info("PKG1 C: %s" % self.python_requires["pkg1"].conanfile.channel) - self.output.info("PKG1 : %s" % self.python_requires["pkg1"].module.myvar) - self.output.info("PKG2 : %s" % self.python_requires["pkg2"].module.myvar) - self.output.info("PKG1F : %s" % self.python_requires["pkg1"].module.myfunct()) - self.output.info("PKG2F : %s" % self.python_requires["pkg2"].module.myfunct()) + self.output.info("PKG1 N: %s" % self.python_requires["pkg1"].conanfile.name)\ + .info("PKG1 V: %s" % self.python_requires["pkg1"].conanfile.version)\ + .info("PKG1 U: %s" % self.python_requires["pkg1"].conanfile.user)\ + .info("PKG1 C: %s" % self.python_requires["pkg1"].conanfile.channel)\ + .info("PKG1 : %s" % self.python_requires["pkg1"].module.myvar)\ + .info("PKG2 : %s" % self.python_requires["pkg2"].module.myvar)\ + .info("PKG1F : %s" % self.python_requires["pkg1"].module.myfunct())\ + .info("PKG2F : %s" % self.python_requires["pkg2"].module.myfunct()) """) client.save({"conanfile.py": conanfile}) client.run("create . --name=consumer --version=0.1 --user=user --channel=testing") diff --git a/conans/test/unittests/client/conan_output_test.py b/conans/test/unittests/client/conan_output_test.py index 351f0eb4b8d..36cb4a1c96a 100644 --- a/conans/test/unittests/client/conan_output_test.py +++ b/conans/test/unittests/client/conan_output_test.py @@ -51,3 +51,13 @@ def test_output_forced_but_conan_logger(): assert out.color is False init.assert_not_called() + + +def test_output_chainable(): + try: + ConanOutput(scope="My package")\ + .title("My title")\ + .highlight("Worked")\ + .info("But there was more that needed to be said") + except Exception as e: + assert False, f"Chained ConanOutput write methods raised an exception {e}" From b088945a91f16278f9f2da097f351ff165556c65 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 27 Dec 2022 23:43:55 +0100 Subject: [PATCH 11/62] [develop2] artifactory_ready (#12792) --- .../download/download_selected_packages_test.py | 2 ++ .../command/download/test_download_patterns.py | 1 + .../integration/command/install/install_test.py | 3 +++ conans/test/integration/command/remove_test.py | 15 +++++++++------ 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/conans/test/integration/command/download/download_selected_packages_test.py b/conans/test/integration/command/download/download_selected_packages_test.py index 61367cf6f3c..1224bdbc57a 100644 --- a/conans/test/integration/command/download/download_selected_packages_test.py +++ b/conans/test/integration/command/download/download_selected_packages_test.py @@ -25,6 +25,7 @@ def setup(): return client, ref, package_ids, str(conanfile) +@pytest.mark.artifactory_ready def test_download_recipe_twice(setup): client, ref, package_ids, conanfile = setup new_client = TestClient(servers=client.servers, inputs=["admin", "password"]) @@ -41,6 +42,7 @@ def test_download_recipe_twice(setup): assert conanfile == load(conanfile_path) +@pytest.mark.artifactory_ready def test_download_packages_twice(setup): client, ref, package_ids, _ = setup new_client = TestClient(servers=client.servers, inputs=["admin", "password"]) diff --git a/conans/test/integration/command/download/test_download_patterns.py b/conans/test/integration/command/download/test_download_patterns.py index e40ba3b0dcc..2813aa910ed 100644 --- a/conans/test/integration/command/download/test_download_patterns.py +++ b/conans/test/integration/command/download/test_download_patterns.py @@ -7,6 +7,7 @@ from conans.util.env import environment_update +@pytest.mark.artifactory_ready class TestDownloadPatterns: # FIXME The fixture is copied from TestUploadPatterns, reuse it @pytest.fixture(scope="class") # Takes 6 seconds, reuse it diff --git a/conans/test/integration/command/install/install_test.py b/conans/test/integration/command/install/install_test.py index b239ed96335..3f9ac7d6d21 100644 --- a/conans/test/integration/command/install/install_test.py +++ b/conans/test/integration/command/install/install_test.py @@ -45,6 +45,7 @@ def test_four_subfolder_install(client): client.run(" install path/to/sub/folder") +@pytest.mark.artifactory_ready def test_install_system_requirements(client): client.save({"conanfile.py": textwrap.dedent(""" from conan import ConanFile @@ -261,6 +262,7 @@ def test_install_anonymous(client): assert "pkg/0.1@lasote/testing: Package installed" in client2.out +@pytest.mark.artifactory_ready def test_install_without_ref(client): client.save({"conanfile.py": GenConanfile("lib", "1.0")}) client.run('create .') @@ -281,6 +283,7 @@ def test_install_without_ref(client): client.run('upload lib/1.0 -c -r default') +@pytest.mark.artifactory_ready def test_install_disabled_remote(client): client.save({"conanfile.py": GenConanfile()}) client.run("create . --name=pkg --version=0.1 --user=lasote --channel=testing") diff --git a/conans/test/integration/command/remove_test.py b/conans/test/integration/command/remove_test.py index 179b73cdb2d..1c911973539 100644 --- a/conans/test/integration/command/remove_test.py +++ b/conans/test/integration/command/remove_test.py @@ -63,10 +63,10 @@ class Test(ConanFile): class RemoveWithoutUserChannel(unittest.TestCase): def setUp(self): - self.test_server = TestServer(users={"lasote": "password"}, - write_permissions=[("lib/1.0@*/*", "lasote")]) + self.test_server = TestServer(users={"admin": "password"}, + write_permissions=[("lib/1.0@*/*", "admin")]) servers = {"default": self.test_server} - self.client = TestClient(servers=servers, inputs=["lasote", "password"]) + self.client = TestClient(servers=servers, inputs=["admin", "password"]) def test_local(self): self.client.save({"conanfile.py": GenConanfile()}) @@ -80,6 +80,7 @@ def test_local(self): self.assertFalse(os.path.exists(ref_layout.base_folder)) self.assertFalse(os.path.exists(pkg_layout.base_folder)) + @pytest.mark.artifactory_ready def test_remote(self): self.client.save({"conanfile.py": GenConanfile()}) self.client.run("create . --name=lib --version=1.0") @@ -102,10 +103,10 @@ class RemovePackageRevisionsTest(unittest.TestCase): NO_SETTINGS_RREF = "4d670581ccb765839f2239cc8dff8fbd" def setUp(self): - self.test_server = TestServer(users={"user": "password"}, - write_permissions=[("foobar/0.1@*/*", "user")]) + self.test_server = TestServer(users={"admin": "password"}, + write_permissions=[("foobar/0.1@*/*", "admin")]) servers = {"default": self.test_server} - self.client = TestClient(servers=servers, inputs=["user", "password"]) + self.client = TestClient(servers=servers, inputs=["admin", "password"]) ref = RecipeReference.loads(f"foobar/0.1@user/testing#{self.NO_SETTINGS_RREF}") self.pref = PkgReference(ref, NO_SETTINGS_PACKAGE_ID, "0ba8627bd47edc3a501e8f0eb9a79e5e") @@ -135,6 +136,7 @@ def test_remove_local_package_id_reference(self): .format(self.NO_SETTINGS_RREF, NO_SETTINGS_PACKAGE_ID)) assert not self.client.package_exists(self.pref) + @pytest.mark.artifactory_ready def test_remove_remote_package_id_reference(self): """ Remove remote package ID based on recipe revision. The package must be deleted, but the recipe must be preserved. @@ -150,6 +152,7 @@ def test_remove_remote_package_id_reference(self): .format(self.NO_SETTINGS_RREF, NO_SETTINGS_PACKAGE_ID)) assert not self.client.package_exists(self.pref) + @pytest.mark.artifactory_ready def test_remove_all_packages_but_the_recipe_at_remote(self): """ Remove all the packages but not the recipe in a remote """ From 7f81902093ff1ee6378fdcbfb04359e3ddab7aac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Wed, 28 Dec 2022 10:08:25 +0100 Subject: [PATCH 12/62] [dev2][conan list] Enhancements (#12781) * Showing latest rrevs and prevs. Improved outputs * wip * Adding new tests * several creates --- conan/api/model.py | 3 +- conan/api/subapi/list.py | 14 ++--- conan/cli/commands/list.py | 34 +++++++---- conan/cli/commands/search.py | 14 +++-- conan/internal/api/select_pattern.py | 12 +++- .../test/integration/command_v2/list_test.py | 56 +++++++++++++++++-- conans/test/utils/tools.py | 1 + 7 files changed, 100 insertions(+), 34 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index fe17391f783..a39f493e59f 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -81,8 +81,9 @@ def prefs(self): return prefs def add_prefs(self, prefs, configurations=None): + confs = configurations or {} for pref in prefs: - binary_info = {} if not configurations else configurations.get(pref) + binary_info = confs.get(pref, {}) self.recipes.setdefault(pref.ref, []).append((pref, binary_info)) def serialize(self): diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index 27419d1b8b6..f8ed0fb6bdd 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -89,17 +89,17 @@ def filter_packages_configurations(self, pkg_configurations, query): return filter_packages(query, pkg_configurations) # TODO: could it be possible to merge this with subapi.search.select? - def select(self, pattern, package_query=None, remote=None): + def select(self, pattern, package_query=None, remote=None, search_mode=None): if package_query and pattern.package_id and "*" not in pattern.package_id: raise ConanException("Cannot specify '-p' package queries, " "if 'package_id' is not a pattern") - list_mode = pattern.mode + search_mode = search_mode or pattern.mode select_bundle = SelectBundle() refs = self.conan_api.search.recipes(pattern.ref, remote=remote) pattern.check_refs(refs) # Show only the recipe references - if list_mode == ListPatternMode.SHOW_REFS: + if search_mode == ListPatternMode.SHOW_REFS: select_bundle.add_refs(refs) return select_bundle @@ -111,7 +111,7 @@ def select(self, pattern, package_query=None, remote=None): rrevs = pattern.filter_rrevs(rrevs) select_bundle.add_refs(rrevs) # Show only the latest recipe revision or all of them - if list_mode in (ListPatternMode.SHOW_ALL_RREVS, ListPatternMode.SHOW_LATEST_RREV): + if search_mode in (ListPatternMode.SHOW_ALL_RREVS, ListPatternMode.SHOW_LATEST_RREV): continue for rrev in rrevs: @@ -124,14 +124,14 @@ def select(self, pattern, package_query=None, remote=None): prefs = pattern.filter_prefs(prefs) # Show all the package IDs and their configurations - if list_mode == ListPatternMode.SHOW_PACKAGE_IDS: + if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: # add pref and its package configuration select_bundle.add_prefs(prefs, configurations=packages) continue for pref in prefs: - if list_mode in (ListPatternMode.SHOW_LATEST_PREV, - ListPatternMode.SHOW_ALL_PREVS): + if search_mode in (ListPatternMode.SHOW_LATEST_PREV, + ListPatternMode.SHOW_ALL_PREVS): if pattern.is_latest_prev: prevs = [self.conan_api.list.latest_package_revision(pref, remote)] else: diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 0f86c8db002..256fec52fb4 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -5,7 +5,7 @@ from conan.cli.command import conan_command, OnceArgument from conan.cli.commands import ConanJSONEncoder from conan.cli.formatters.list import list_packages_html -from conan.internal.api.select_pattern import ListPattern +from conan.internal.api.select_pattern import ListPattern, ListPatternMode from conans.util.dates import timestamp_to_str remote_color = Color.BRIGHT_BLUE @@ -18,9 +18,9 @@ def print_list_text(results): - results = results["results"] - - for remote, refs in results.items(): + info = results["results"] + search_mode = results["search_mode"] + for remote, refs in info.items(): cli_out_write(f"{remote}:", fg=remote_color) if refs.get("error"): @@ -43,13 +43,17 @@ def print_list_text(results): if prefs: for pref, binary_info in prefs: pref_date = f" ({timestamp_to_str(pref.timestamp)})" if pref.timestamp else "" - cli_out_write(f"{indentation * 3}PID: {pref.package_id}{pref_date}", - fg=reference_color) - if not binary_info and pref.revision: - cli_out_write(f"{indentation * 4}PREV: {pref.revision}", fg=field_color) - continue - elif not binary_info: - cli_out_write(f"{indentation * 4}No package info/revision was found.", + if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: + cli_out_write(f"{indentation * 3}PID: {pref.package_id}{pref_date}", + fg=reference_color) + if not binary_info: + cli_out_write(f"{indentation * 4}Empty package information", + fg=field_color) + continue + elif search_mode in (ListPatternMode.SHOW_ALL_PREVS, + ListPatternMode.SHOW_LATEST_PREV): + cli_out_write(f"{indentation * 3}PID: {pref.package_id}", fg=reference_color) + cli_out_write(f"{indentation * 4}PREV: {pref.revision}{pref_date}", fg=field_color) continue for item, contents in binary_info.items(): @@ -62,6 +66,11 @@ def print_list_text(results): else: for c in contents: cli_out_write(f"{indentation * 5}{c}", fg=value_color) + elif search_mode in (ListPatternMode.SHOW_PACKAGE_IDS, + ListPatternMode.SHOW_ALL_PREVS, + ListPatternMode.SHOW_LATEST_PREV): + cli_out_write(f"{indentation * 3}There are no packages for this revision.", + fg=field_color) def print_list_json(data): @@ -89,6 +98,7 @@ def list(conan_api: ConanAPI, parser, *args): parser.add_argument("-c", "--cache", action='store_true', help="Search in the local cache") args = parser.parse_args(*args) + ref_pattern = ListPattern(args.reference) # If neither remote nor cache are defined, show results only from cache remotes = [] if args.cache or not args.remote: @@ -98,7 +108,6 @@ def list(conan_api: ConanAPI, parser, *args): results = {} for remote in remotes: name = getattr(remote, "name", "Local Cache") - ref_pattern = ListPattern(args.reference) try: list_bundle = conan_api.list.select(ref_pattern, args.package_query, remote) except Exception as e: @@ -108,5 +117,6 @@ def list(conan_api: ConanAPI, parser, *args): else list_bundle.recipes return { "results": results, + "search_mode": ref_pattern.mode, "conan_api": conan_api } diff --git a/conan/cli/commands/search.py b/conan/cli/commands/search.py index 0f00b55bf76..f3c378438d9 100644 --- a/conan/cli/commands/search.py +++ b/conan/cli/commands/search.py @@ -3,11 +3,11 @@ from conan.api.conan_api import ConanAPI from conan.cli.command import conan_command from conan.cli.commands.list import print_list_text, print_list_json -from conan.internal.api.select_pattern import ListPattern +from conan.internal.api.select_pattern import ListPattern, ListPatternMode from conans.errors import ConanException -# FIXME: "conan search" == "conan list recipes -r="*" -c" --> implement @conan_alias_command?? +# FIXME: "conan search" == "conan list (*) -r="*"" --> implement @conan_alias_command?? @conan_command(group="Consumer", formatters={"text": print_list_text, "json": print_list_json}) def search(conan_api: ConanAPI, parser, *args): @@ -20,6 +20,8 @@ def search(conan_api: ConanAPI, parser, *args): help="Remote names. Accepts wildcards. If not specified it searches " "in all the remotes") args = parser.parse_args(*args) + ref_pattern = ListPattern(args.reference) + search_mode = ListPatternMode.SHOW_REFS remotes = conan_api.remotes.list(args.remote) if not remotes: @@ -27,14 +29,16 @@ def search(conan_api: ConanAPI, parser, *args): results = OrderedDict() for remote in remotes: - ref_pattern = ListPattern(args.reference) try: - list_bundle = conan_api.list.select(ref_pattern, package_query=None, remote=remote) + list_bundle = conan_api.list.select(ref_pattern, package_query=None, + remote=remote, search_mode=search_mode) except Exception as e: results[remote.name] = {"error": str(e)} else: - results[remote.name] = list_bundle.serialize() if args.format == "json" else list_bundle.recipes + results[remote.name] = list_bundle.serialize() if args.format == "json" \ + else list_bundle.recipes return { "results": results, + "search_mode": search_mode, "conan_api": conan_api } diff --git a/conan/internal/api/select_pattern.py b/conan/internal/api/select_pattern.py index 5305d2475a7..17c55c8cb1d 100644 --- a/conan/internal/api/select_pattern.py +++ b/conan/internal/api/select_pattern.py @@ -87,7 +87,7 @@ def mode(self): # zlib:PID -> it will fail on server side! Nothing to do here else: if self.package_id is None: - # zlib/1.2.11 + # zlib/1.2.11 | zlib/1.2.11#latest if self.is_latest_rrev: return ListPatternMode.SHOW_LATEST_RREV # zlib/1.2.11#RREV @@ -100,15 +100,21 @@ def mode(self): else: if self.package_id is None: # zlib/* | zlib* - if self.is_latest_rrev: + if self.is_latest_rrev and "#latest" not in self.raw: return ListPatternMode.SHOW_REFS + # zlib/*#latest + elif "#latest" in self.raw: + return ListPatternMode.SHOW_LATEST_RREV # zlib/1.2.11#* elif "*" in self.rrev: return ListPatternMode.SHOW_ALL_RREVS else: # zlib/1.2.11#latest:* | zlib/1.2.11#RREV:* - if self.is_latest_prev: + if self.is_latest_prev and "#latest" not in self.raw: return ListPatternMode.SHOW_PACKAGE_IDS + # zlib/1.2.11:*#latest + elif "#latest" in self.raw: + return ListPatternMode.SHOW_LATEST_PREV # zlib/1.2.11#latest:*#* | zlib/1.2.11#RREV:PID#* elif "*" in self.prev: return ListPatternMode.SHOW_ALL_PREVS diff --git a/conans/test/integration/command_v2/list_test.py b/conans/test/integration/command_v2/list_test.py index 69300f7d904..6b345e03239 100644 --- a/conans/test/integration/command_v2/list_test.py +++ b/conans/test/integration/command_v2/list_test.py @@ -178,7 +178,7 @@ def test_list_recipes(self): """) assert expected_output == self.client.out - def test_list_latest_recipe_revision(self): + def test_list_latest_recipe_revision_by_default(self): self.client.save({ "conanfile.py": GenConanfile("test_recipe", "1.0.0").with_package_file("file.h", "0.1") }) @@ -192,7 +192,28 @@ def test_list_latest_recipe_revision(self): """ % rrev.repr_notime()) assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - def test_list_latest_package_revisions(self): + def test_list_all_the_latest_recipe_revision(self): + self.client.save({ + "hello1.py": GenConanfile("hello", "1.0.0").with_generator("CMakeToolchain"), # rrev + "hello.py": GenConanfile("hello", "1.0.0"), # latest rrev + "bye.py": GenConanfile("bye", "1.0.0") + }) + self.client.run("export hello1.py --user=user --channel=channel") + self.client.run("export hello.py --user=user --channel=channel") + hello_latest_rrev = self._get_lastest_recipe_ref("hello/1.0.0@user/channel") + self.client.run("export bye.py --user=user --channel=channel") + self.client.run(f"list *#latest") + expected_output = textwrap.dedent(f"""\ + Local Cache: + bye + bye/1.0.0@user/channel#c720a82a9c904a0450ec1aa177281ea2 .* + hello + hello/1.0.0@user/channel#7a34833afbd87d791b2201882b1afb2b .* + """) + assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) + assert hello_latest_rrev.repr_notime() in expected_output + + def test_list_latest_package_revisions_by_default(self): self.client.save({ "conanfile.py": GenConanfile("test_recipe", "1.0.0").with_package_file("file.h", "0.1") }) @@ -205,12 +226,35 @@ def test_list_latest_package_revisions(self): Local Cache: test_recipe test_recipe/1.0.0@user/channel#ddfadce26d00a560850eb8767fe76ae4 .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 .* - PREV: 9c929aed65f04337a4143311d72fc897 + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 9c929aed65f04337a4143311d72fc897 .* + """) + assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) + + def test_list_all_the_latest_package_revisions(self): + self.client.save({ + "hello.py": GenConanfile("hello", "1.0.0").with_package_file("file.h", "0.1"), + "bye.py": GenConanfile("bye", "1.0.0").with_package_file("file.h", "0.1") + }) + self.client.run("create hello.py --user=user --channel=channel") + self.client.run("create hello.py --user=user --channel=channel") # latest prev + self.client.run("create bye.py --user=user --channel=channel") + self.client.run("create bye.py --user=user --channel=channel") # latest prev + self.client.run("list *:*#latest") + expected_output = textwrap.dedent(f"""\ + Local Cache: + bye + bye/1.0.0@user/channel#51edd97e27e407a01be830282558c32a .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 9c929aed65f04337a4143311d72fc897 .* + hello + hello/1.0.0@user/channel#6fccfa5dd0bbb1223578c1771839eb6d .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 9c929aed65f04337a4143311d72fc897 .* """) assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - def test_search_with_full_reference_but_package_has_no_properties(self): + def test_search_package_ids_but_empty_conan_info(self): remote_name = "remote1" recipe_name = "test_recipe/1.0.0@user/channel" self._add_remote(remote_name) @@ -222,7 +266,7 @@ def test_search_with_full_reference_but_package_has_no_properties(self): test_recipe test_recipe/1.0.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - No package info/revision was found. + Empty package information """) assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index cda349361ee..9faba630727 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -808,6 +808,7 @@ def package_revision(self, pref): latest_prev = self.cache.get_latest_package_reference(tmp) return latest_prev.revision + # FIXME: 2.0: adapt this function to using the new "conan list xxxx" and recover the xfail tests def search(self, pattern, remote=None, assert_error=False, args=None): remote = " -r={}".format(remote) if remote else "" self.run("search {} --json {} {} {}".format(pattern, ".tmp.json", remote, From 77059831e53b0a5ca3bc09153a57a9150e62846b Mon Sep 17 00:00:00 2001 From: James Date: Wed, 28 Dec 2022 12:08:33 +0100 Subject: [PATCH 13/62] [develop2] Forbid access to .info in package_info() (#12775) --- conans/client/installer.py | 5 +++-- .../integration/conanfile/test_attributes_scope.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/conans/client/installer.py b/conans/client/installer.py index a914fbf847b..36c92ceaa2e 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -12,7 +12,7 @@ from conans.client.graph.install_graph import InstallGraph from conans.client.source import retrieve_exports_sources, config_source from conans.errors import (ConanException, ConanExceptionInUserConanfileMethod, - conanfile_exception_formatter) + conanfile_exception_formatter, conanfile_remove_attr) from conans.model.build_info import CppInfo from conans.model.package_ref import PkgReference from conans.paths import CONANINFO @@ -353,7 +353,8 @@ def _call_package_info(self, conanfile, package_folder, is_editable): self._hook_manager.execute("pre_package_info", conanfile=conanfile) if hasattr(conanfile, "package_info"): - conanfile.package_info() + with conanfile_remove_attr(conanfile, ['info'], "package_info"): + conanfile.package_info() # TODO: Check this package_folder usage for editable when not defined conanfile.cpp.package.set_relative_base_folder(package_folder) diff --git a/conans/test/integration/conanfile/test_attributes_scope.py b/conans/test/integration/conanfile/test_attributes_scope.py index 506c21ddf57..a8637ee1048 100644 --- a/conans/test/integration/conanfile/test_attributes_scope.py +++ b/conans/test/integration/conanfile/test_attributes_scope.py @@ -50,6 +50,20 @@ def package_id(self): t.run('create . --name=name --version=version', assert_error=True) assert "'self.options' access in 'package_id()' method is forbidden" in t.out + def test_info_not_in_package_info(self): + t = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + + class Recipe(ConanFile): + + def package_info(self): + self.info + """) + t.save({'conanfile.py': conanfile}) + t.run('create . --name=name --version=version', assert_error=True) + assert "'self.info' access in 'package_info()' method is forbidden" in t.out + def test_info_not_in_package(self): # self.info is not available in 'package' t = TestClient() From 248be244a4db9904985fe98077b04d5e52c40433 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 28 Dec 2022 12:10:17 +0100 Subject: [PATCH 14/62] [develop2] tests to check access to info in package_id() (#12776) --- .../integration/package_id/package_id_test.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/conans/test/integration/package_id/package_id_test.py b/conans/test/integration/package_id/package_id_test.py index be1dcad56a5..6eb01452db9 100644 --- a/conans/test/integration/package_id/package_id_test.py +++ b/conans/test/integration/package_id/package_id_test.py @@ -143,3 +143,44 @@ def package_id(self): '-s compiler.version=190 -s build_type=Debug -s compiler.runtime=dynamic') client.assert_listed_binary({"pkg/0.1": (package_id, "Cache")}) + +def test_package_id_requires_info(): + """ if we dont restrict ``package_id()`` to use only ``self.info`` it will do nothing and fail + if we ``del self.settings.arch`` instead of ``del self.info.settings.arch`` + https://github.com/conan-io/conan/issues/12693 + """ + conanfile = textwrap.dedent(""" + from conan import ConanFile + class TestConan(ConanFile): + settings = "os", "arch" + + def package_id(self): + if self.info.settings.os == "Windows": + del self.info.settings.arch + """) + client = TestClient() + client.save({"conanfile.py": conanfile}) + client.run("create . --name=pkg --version=0.1 -s os=Windows -s arch=armv8") + client.assert_listed_binary({"pkg/0.1": ("ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715", "Build")}) + client.run("create . --name=pkg --version=0.1 -s os=Windows -s arch=x86_64") + client.assert_listed_binary({"pkg/0.1": ("ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715", "Build")}) + + +def test_package_id_validate_settings(): + """ ``self.info`` has some validation, the first time it executes + https://github.com/conan-io/conan/issues/12693 + """ + conanfile = textwrap.dedent(""" + from conan import ConanFile + class TestConan(ConanFile): + settings = "os", "arch" + + def package_id(self): + if self.info.settings.os == "DONT_EXIST": + del self.info.settings.arch + """) + c = TestClient() + c.save({"conanfile.py": conanfile}) + c.run("create . --name=pkg --version=0.1", assert_error=True) + print(c.out) + assert "ConanException: Invalid setting 'DONT_EXIST' is not a valid 'settings.os' value" in c.out From d62eccf751f88543c3a0630e7361f98fb11a7e36 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 28 Dec 2022 13:04:23 +0100 Subject: [PATCH 15/62] [develop2] deprecations in conf (#12797) * [develop2] deprecations in conf * fix test --- conan/tools/gnu/autotoolstoolchain.py | 2 +- conans/model/conf.py | 50 ++----------------- .../configuration/conf/test_conf_from_br.py | 2 +- .../configuration/conf/test_conf_profile.py | 2 +- .../tools/system/package_manager_test.py | 16 +++--- .../unittests/tools/microsoft/test_msbuild.py | 2 +- 6 files changed, 15 insertions(+), 59 deletions(-) diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index 338c282bba9..add80002550 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -81,7 +81,7 @@ def __init__(self, conanfile, namespace=None, prefix="/"): # Apple Stuff if os_build == "Macos": # SDK path is mandatory for cross-building - sdk_path = conanfile.conf["tools.apple:sdk_path"] + sdk_path = conanfile.conf.get("tools.apple:sdk_path") if not sdk_path: raise ConanException("You must provide a valid SDK path for cross-compilation.") apple_arch = to_apple_arch(self._conanfile) diff --git a/conans/model/conf.py b/conans/model/conf.py index c7829ae3a7e..033f60ed77c 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -119,6 +119,8 @@ class _ConfVarPlaceHolder: class _ConfValue(object): def __init__(self, name, value, path=False): + if name != name.lower(): + raise ConanException("Conf '{}' must be lowercase".format(name)) self._name = name self._value = value self._value_type = type(value) @@ -246,34 +248,11 @@ def __eq__(self, other): """ return other._values == self._values - def __getitem__(self, name): - """ - DEPRECATED: it's going to disappear in Conan 2.0. Use self.get() instead. - """ - # FIXME: Keeping backward compatibility - return self.get(name) - - def __setitem__(self, name, value): - """ - DEPRECATED: it's going to disappear in Conan 2.0. - """ - # FIXME: Keeping backward compatibility - self.define(name, value) # it's like a new definition - def items(self): # FIXME: Keeping backward compatibility for k, v in self._values.items(): yield k, v.value - @staticmethod - def _get_boolean_value(value): - if type(value) is bool: - return value - elif str(value).lower() in Conf.boolean_false_expressions: - return False - else: - return True - def get(self, conf_name, default=None, check_type=None): """ Get all the values of the given configuration name. @@ -294,7 +273,7 @@ def get(self, conf_name, default=None, check_type=None): # Some smart conversions if check_type is bool and not isinstance(v, bool): # Perhaps, user has introduced a "false", "0" or even "off" - return self._get_boolean_value(v) + return str(v).lower() not in Conf.boolean_false_expressions elif check_type is str and not isinstance(v, str): return str(v) elif v is None: # value was unset @@ -319,11 +298,6 @@ def pop(self, conf_name, default=None): self._values.pop(conf_name, None) return value - @staticmethod - def _validate_lower_case(name): - if name != name.lower(): - raise ConanException("Conf '{}' must be lowercase".format(name)) - def copy(self): c = Conf() c._values = self._values.copy() @@ -351,11 +325,9 @@ def define(self, name, value): :param name: Name of the configuration. :param value: Value of the configuration. """ - self._validate_lower_case(name) self._values[name] = _ConfValue(name, value) def define_path(self, name, value): - self._validate_lower_case(name) self._values[name] = _ConfValue(name, value, path=True) def unset(self, name): @@ -373,12 +345,10 @@ def update(self, name, value): :param name: Name of the configuration. :param value: Value of the configuration. """ - self._validate_lower_case(name) conf_value = _ConfValue(name, {}) self._values.setdefault(name, conf_value).update(value) def update_path(self, name, value): - self._validate_lower_case(name) conf_value = _ConfValue(name, {}, path=True) self._values.setdefault(name, conf_value).update(value) @@ -389,12 +359,10 @@ def append(self, name, value): :param name: Name of the configuration. :param value: Value to append. """ - self._validate_lower_case(name) conf_value = _ConfValue(name, [_ConfVarPlaceHolder]) self._values.setdefault(name, conf_value).append(value) def append_path(self, name, value): - self._validate_lower_case(name) conf_value = _ConfValue(name, [_ConfVarPlaceHolder], path=True) self._values.setdefault(name, conf_value).append(value) @@ -405,12 +373,10 @@ def prepend(self, name, value): :param name: Name of the configuration. :param value: Value to prepend. """ - self._validate_lower_case(name) conf_value = _ConfValue(name, [_ConfVarPlaceHolder]) self._values.setdefault(name, conf_value).prepend(value) def prepend_path(self, name, value): - self._validate_lower_case(name) conf_value = _ConfValue(name, [_ConfVarPlaceHolder], path=True) self._values.setdefault(name, conf_value).prepend(value) @@ -604,16 +570,6 @@ def update(self, key, value, profile=False, method="define"): # Update self._update_conf_definition(pattern, conf) - def as_list(self): - result = [] - for pattern, conf in self._pattern_confs.items(): - for name, value in sorted(conf.items()): - if pattern: - result.append(("{}:{}".format(pattern, name), value)) - else: - result.append((name, value)) - return result - def dumps(self): result = [] for pattern, conf in self._pattern_confs.items(): diff --git a/conans/test/integration/configuration/conf/test_conf_from_br.py b/conans/test/integration/configuration/conf/test_conf_from_br.py index 601f7e25a44..a67ffd565e1 100644 --- a/conans/test/integration/configuration/conf/test_conf_from_br.py +++ b/conans/test/integration/configuration/conf/test_conf_from_br.py @@ -11,7 +11,7 @@ def test_basic(): class Pkg(ConanFile): def package_info(self): - self.conf_info["tools.android:ndk_path"] = "MY-NDK!!!" + self.conf_info.define_path("tools.android:ndk_path", "MY-NDK!!!") """) client.save({"conanfile.py": conanfile}) client.run("create . --name=android_ndk --version=1.0") diff --git a/conans/test/integration/configuration/conf/test_conf_profile.py b/conans/test/integration/configuration/conf/test_conf_profile.py index b6c2dda5c9a..e5c2686a40e 100644 --- a/conans/test/integration/configuration/conf/test_conf_profile.py +++ b/conans/test/integration/configuration/conf/test_conf_profile.py @@ -191,7 +191,7 @@ def test_conf_package_patterns(): conanfile = GenConanfile() generate = """ def generate(self): - value = self.conf["user.build:myconfig"] + value = self.conf.get("user.build:myconfig") self.output.warning("{} Config:{}".format(self.ref.name, value)) """ client.save({"dep/conanfile.py": str(conanfile) + generate, diff --git a/conans/test/integration/tools/system/package_manager_test.py b/conans/test/integration/tools/system/package_manager_test.py index eacb1e9fc92..07f7bdf02b6 100644 --- a/conans/test/integration/tools/system/package_manager_test.py +++ b/conans/test/integration/tools/system/package_manager_test.py @@ -146,9 +146,9 @@ def test_tools_update_mode_install(tool_class, result): conanfile = ConanFileMock() conanfile.conf = Conf() conanfile.settings = Settings() - conanfile.conf["tools.system.package_manager:tool"] = tool_class.tool_name + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) for mode in ["check", "install"]: - conanfile.conf["tools.system.package_manager:mode"] = mode + conanfile.conf.define("tools.system.package_manager:mode", mode) with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" tool = tool_class(conanfile) @@ -169,8 +169,8 @@ def test_dnf_yum_return_code_100(tool_class, result): conanfile = ConanFileMock() conanfile.conf = Conf() conanfile.settings = Settings() - conanfile.conf["tools.system.package_manager:tool"] = tool_class.tool_name - conanfile.conf["tools.system.package_manager:mode"] = "install" + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" tool = tool_class(conanfile) @@ -223,8 +223,8 @@ def test_tools_install_mode_install_different_archs(tool_class, arch_host, resul conanfile.conf = Conf() conanfile.settings = MockSettings({"arch": f"{arch_host}"}) conanfile.settings_build = MockSettings({"arch": "x86_64"}) - conanfile.conf["tools.system.package_manager:tool"] = tool_class.tool_name - conanfile.conf["tools.system.package_manager:mode"] = "install" + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" tool = tool_class(conanfile) @@ -251,8 +251,8 @@ def test_tools_install_archless(tool_class, result): conanfile.conf = Conf() conanfile.settings = MockSettings({"arch": "x86"}) conanfile.settings_build = MockSettings({"arch": "x86_64"}) - conanfile.conf["tools.system.package_manager:tool"] = tool_class.tool_name - conanfile.conf["tools.system.package_manager:mode"] = "install" + conanfile.conf.define("tools.system.package_manager:tool", tool_class.tool_name) + conanfile.conf.define("tools.system.package_manager:mode", "install") with mock.patch('conan.ConanFile.context', new_callable=PropertyMock) as context_mock: context_mock.return_value = "host" tool = tool_class(conanfile, arch_names={}) diff --git a/conans/test/unittests/tools/microsoft/test_msbuild.py b/conans/test/unittests/tools/microsoft/test_msbuild.py index ad87baf2e1e..987c77e8980 100644 --- a/conans/test/unittests/tools/microsoft/test_msbuild.py +++ b/conans/test/unittests/tools/microsoft/test_msbuild.py @@ -247,7 +247,7 @@ def test_msbuildtoolchain_changing_flags_via_attributes(): conanfile.settings = settings conanfile.folders.set_base_generators(test_folder) conanfile.conf = Conf() - conanfile.conf["tools.microsoft.msbuild:installation_path"] = "." + conanfile.conf.define_path("tools.microsoft.msbuild:installation_path", ".") conanfile.settings_build = settings conanfile.settings.build_type = "Release" conanfile.settings.compiler = "msvc" From 79100038f289553bbe09c9543a7080936ded4e27 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 28 Dec 2022 23:38:39 +0100 Subject: [PATCH 16/62] do not allow mixing legacy helpers with new generators (#12795) * do not allow mixing legacy helpers with new generators * add test * fix tests --- conan/tools/cmake/cmakedeps/cmakedeps.py | 1 + conan/tools/cmake/toolchain/toolchain.py | 1 + conan/tools/microsoft/msbuilddeps.py | 1 + conan/tools/microsoft/toolchain.py | 1 + conans/client/build/cmake.py | 3 + conans/client/build/msbuild.py | 4 + .../cmake/cmakedeps/test_apple_frameworks.py | 6 +- .../cmakedeps/test_cmakedeps_build_modules.py | 6 +- .../toolchains/test_not_mix_legacy.py | 79 +++++++++++++++++++ 9 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 conans/test/integration/toolchains/test_not_mix_legacy.py diff --git a/conan/tools/cmake/cmakedeps/cmakedeps.py b/conan/tools/cmake/cmakedeps/cmakedeps.py index 1a873729a77..47647a715af 100644 --- a/conan/tools/cmake/cmakedeps/cmakedeps.py +++ b/conan/tools/cmake/cmakedeps/cmakedeps.py @@ -17,6 +17,7 @@ class CMakeDeps(object): def __init__(self, conanfile): self._conanfile = conanfile + self._conanfile.must_use_new_helpers = True # TODO: Remove 2.0 self.arch = self._conanfile.settings.get_safe("arch") self.configuration = str(self._conanfile.settings.build_type) diff --git a/conan/tools/cmake/toolchain/toolchain.py b/conan/tools/cmake/toolchain/toolchain.py index cd54a9a8776..2a0e181f804 100644 --- a/conan/tools/cmake/toolchain/toolchain.py +++ b/conan/tools/cmake/toolchain/toolchain.py @@ -118,6 +118,7 @@ class CMakeToolchain(object): def __init__(self, conanfile, generator=None): self._conanfile = conanfile + self._conanfile.must_use_new_helpers = True # TODO: Remove 2.0 self.generator = self._get_generator(generator) self.variables = Variables() # This doesn't support multi-config, they go to the same configPreset common in multi-config diff --git a/conan/tools/microsoft/msbuilddeps.py b/conan/tools/microsoft/msbuilddeps.py index 2053d38a033..8e0ab92dcda 100644 --- a/conan/tools/microsoft/msbuilddeps.py +++ b/conan/tools/microsoft/msbuilddeps.py @@ -90,6 +90,7 @@ class MSBuildDeps(object): def __init__(self, conanfile): self._conanfile = conanfile + self._conanfile.must_use_new_helpers = True # TODO: Remove 2.0 self.configuration = conanfile.settings.build_type # TODO: This platform is not exactly the same as ``msbuild_arch``, because it differs # in x86=>Win32 diff --git a/conan/tools/microsoft/toolchain.py b/conan/tools/microsoft/toolchain.py index 2b9c4686b93..b98f6be4560 100644 --- a/conan/tools/microsoft/toolchain.py +++ b/conan/tools/microsoft/toolchain.py @@ -45,6 +45,7 @@ class MSBuildToolchain(object): def __init__(self, conanfile): self._conanfile = conanfile + self._conanfile.must_use_new_helpers = True # TODO: Remove 2.0 self.preprocessor_definitions = {} self.compile_options = {} self.cxxflags = [] diff --git a/conans/client/build/cmake.py b/conans/client/build/cmake.py index acaeffb3cf5..0006d862268 100644 --- a/conans/client/build/cmake.py +++ b/conans/client/build/cmake.py @@ -46,6 +46,9 @@ def __init__(self, conanfile, generator=None, cmake_system_name=True, :param cmake_program: Path to the custom cmake executable :param generator_platform: Generator platform name or none to autodetect (-A cmake option) """ + if getattr(conanfile, "must_use_new_helpers", None): + raise ConanException("Using the wrong 'CMake' helper. To use CMakeDeps, CMakeToolchain " + "you should use 'from conan.tools.cmake import CMake'") self._append_vcvars = append_vcvars self._conanfile = conanfile self._settings = conanfile.settings diff --git a/conans/client/build/msbuild.py b/conans/client/build/msbuild.py index da88f07037b..778b81a7628 100644 --- a/conans/client/build/msbuild.py +++ b/conans/client/build/msbuild.py @@ -22,6 +22,10 @@ class MSBuild(object): def __init__(self, conanfile): if isinstance(conanfile, ConanFile): + if getattr(conanfile, "must_use_new_helpers", None): + raise ConanException( + "Using the wrong 'MSBuild' helper. To use MSBuildDeps, MSBuildToolchain " + "you should use 'from conan.tools.microsoft import MSBuild'") self._conanfile = conanfile self._settings = self._conanfile.settings self._output = self._conanfile.output diff --git a/conans/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py b/conans/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py index 33cb47b0352..7dbc10bbff1 100644 --- a/conans/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py +++ b/conans/test/functional/toolchains/cmake/cmakedeps/test_apple_frameworks.py @@ -348,7 +348,8 @@ def test(self): @pytest.mark.skipif(platform.system() != "Darwin", reason="Only OSX") def test_component_uses_apple_framework(): conanfile_py = textwrap.dedent(""" -from conans import ConanFile, CMake, tools +from conans import ConanFile, tools +from conan.tools.cmake import CMake class HelloConan(ConanFile): @@ -406,7 +407,8 @@ def package_info(self): """) test_conanfile_py = textwrap.dedent(""" import os -from conans import ConanFile, CMake, tools +from conans import ConanFile, tools +from conan.tools.cmake import CMake class TestPackageConan(ConanFile): settings = "os", "compiler", "build_type", "arch" diff --git a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_build_modules.py b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_build_modules.py index 3566bfaded3..ae6a392b331 100644 --- a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_build_modules.py +++ b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_build_modules.py @@ -34,7 +34,8 @@ def package_info(self): client.run("create .") consumer = textwrap.dedent(""" - from conans import ConanFile, CMake + from conans import ConanFile + from conan.tools.cmake import CMake class Conan(ConanFile): name = "consumer" @@ -103,7 +104,8 @@ def package_info(self): client.run("create .") consumer = textwrap.dedent(""" - from conans import ConanFile, CMake + from conans import ConanFile + from conan.tools.cmake import CMake class Conan(ConanFile): name = "consumer" diff --git a/conans/test/integration/toolchains/test_not_mix_legacy.py b/conans/test/integration/toolchains/test_not_mix_legacy.py new file mode 100644 index 00000000000..c0fb38fec7a --- /dev/null +++ b/conans/test/integration/toolchains/test_not_mix_legacy.py @@ -0,0 +1,79 @@ +import textwrap + +from conans.test.utils.tools import TestClient + + +def test_not_mix_legacy_cmake(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans import CMake + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "build_type" + generators = "CMakeDeps" + def build(self): + CMake(self) + """) + c.save({"conanfile.py": conanfile}) + c.run("create .", assert_error=True) + assert "Using the wrong 'CMake' helper" in c.out + + +def test_not_mix_legacy_cmake_generate(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans import CMake + from conan.tools.cmake import CMakeDeps + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "build_type" + def generate(self): + CMakeDeps(self) + def build(self): + CMake(self) + """) + c.save({"conanfile.py": conanfile}) + c.run("create .", assert_error=True) + assert "Using the wrong 'CMake' helper" in c.out + + +def test_not_mix_legacy_msbuild(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans import MSBuild + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "build_type", "arch" + generators = "MSBuildDeps" + def build(self): + MSBuild(self) + """) + c.save({"conanfile.py": conanfile}) + c.run("create .", assert_error=True) + assert "Using the wrong 'MSBuild' helper" in c.out + + +def test_not_mix_legacy_msbuild_generate(): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conans import MSBuild + from conan.tools.microsoft import MSBuildDeps + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + settings = "build_type", "arch" + def generate(self): + MSBuildDeps(self) + def build(self): + MSBuild(self) + """) + c.save({"conanfile.py": conanfile}) + c.run("create .", assert_error=True) + assert "Using the wrong 'MSBuild' helper" in c.out From 56d7221634a4a9c314038343328fcf25abd7655e Mon Sep 17 00:00:00 2001 From: James Date: Thu, 29 Dec 2022 00:39:05 +0100 Subject: [PATCH 17/62] fix gcc 12 cppstd (#12796) --- conan/tools/_compilers.py | 4 ++++ .../unittests/tools/gnu/autotoolschain_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/conan/tools/_compilers.py b/conan/tools/_compilers.py index a4f6f9da29d..fb976a469fd 100644 --- a/conan/tools/_compilers.py +++ b/conan/tools/_compilers.py @@ -398,6 +398,10 @@ def _cppstd_gcc(gcc_version, cppstd): v23 = "c++2b" vgnu23 = "gnu++2b" + if Version(gcc_version) >= "12": + v20 = "c++20" + vgnu20 = "gnu++20" + flag = {"98": v98, "gnu98": vgnu98, "11": v11, "gnu11": vgnu11, "14": v14, "gnu14": vgnu14, diff --git a/conans/test/unittests/tools/gnu/autotoolschain_test.py b/conans/test/unittests/tools/gnu/autotoolschain_test.py index 8141dc8c870..93b7d31eca0 100644 --- a/conans/test/unittests/tools/gnu/autotoolschain_test.py +++ b/conans/test/unittests/tools/gnu/autotoolschain_test.py @@ -25,6 +25,23 @@ def test_get_gnu_triplet_for_cross_building(): assert autotoolschain._build == "i686-solaris" +def test_get_toolchain_cppstd(): + settings = MockSettings({"build_type": "Release", + "compiler": "gcc", + "compiler.version": "10", + "compiler.cppstd": "20", + "os": "Linux", + "arch": "x86_64"}) + conanfile = ConanFileMock() + conanfile.settings = settings + conanfile.settings_build = settings + autotoolschain = AutotoolsToolchain(conanfile) + assert autotoolschain.cppstd == "-std=c++2a" + settings.values["compiler.version"] = "12" + autotoolschain = AutotoolsToolchain(conanfile) + assert autotoolschain.cppstd == "-std=c++20" + + @pytest.mark.parametrize("runtime, runtime_type, expected", [("static", "Debug", "MTd"), ("static", "Release", "MT"), From 1dfc0326097c20c7ceb024dcfe1867eca900a28c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 29 Dec 2022 12:36:28 +0100 Subject: [PATCH 18/62] [develop2] test for config home and forbidden errors (#12806) --- .../test/integration/command/config_test.py | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/conans/test/integration/command/config_test.py b/conans/test/integration/command/config_test.py index a089cf5b92f..6805232da1d 100644 --- a/conans/test/integration/command/config_test.py +++ b/conans/test/integration/command/config_test.py @@ -1,8 +1,8 @@ import json import os +from conan.api.conan_api import ConanAPI from conans.model.conf import BUILT_IN_CONFS -from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient from conans.util.env import environment_update @@ -16,35 +16,23 @@ def test_missing_subarguments(): assert "ERROR: Exiting with code: 2" in client.out -def test_config_home_default(): - """ config home MUST show conan home path +class TestConfigHome: + """ The test framework cannot test the CONAN_HOME env-var because it is not using it + (it will break tests for maintainers that have the env-var defined) """ - client = TestClient() - client.run("config home") - assert f"{client.cache.cache_folder}\n" == client.stdout - - -def test_config_home_custom_home_dir(): - """ config home MUST accept CONAN_HOME as custom home path - """ - cache_folder = os.path.join(temp_folder(), "custom") - with environment_update({"CONAN_HOME": cache_folder}): - client = TestClient(cache_folder=cache_folder) + def test_config_home_default(self): + client = TestClient() client.run("config home") - assert cache_folder in client.out - client.run("config home --format=text") assert f"{client.cache.cache_folder}\n" == client.stdout + client.run("config home --format=text") + assert f"{client.cache.cache_folder}\n" == client.stdout -def test_config_home_custom_install(): - """ config install MUST accept CONAN_HOME as custom home path - """ - cache_folder = os.path.join(temp_folder(), "custom") - with environment_update({"CONAN_HOME": cache_folder}): - client = TestClient(cache_folder=cache_folder) - client.save({"conanfile.py": GenConanfile()}) - client.run("install .") - assert "Installing (downloading, building) binaries" in client.out + def test_api_uses_env_var_home(self): + cache_folder = os.path.join(temp_folder(), "custom") + with environment_update({"CONAN_HOME": cache_folder}): + api = ConanAPI() + assert api.cache_folder == cache_folder def test_config_list(): From 5d08795b56a3b14455a632dc3792aff5397eaace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Thu, 29 Dec 2022 13:42:32 +0100 Subject: [PATCH 19/62] [dev2][conan list] Enhancements 2.0 (#12801) * wip * Added more tests * fix * removed useless settings --- conan/api/model.py | 15 ++- conan/api/subapi/list.py | 20 ++-- conan/cli/commands/list.py | 86 +++++++------- conan/cli/commands/search.py | 2 +- .../test/integration/command_v2/list_test.py | 106 ++++++++++++++++++ 5 files changed, 174 insertions(+), 55 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index a39f493e59f..874d6219f60 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -81,15 +81,22 @@ def prefs(self): return prefs def add_prefs(self, prefs, configurations=None): - confs = configurations or {} + confs = configurations or OrderedDict() for pref in prefs: - binary_info = confs.get(pref, {}) + binary_info = confs.get(pref, OrderedDict()) self.recipes.setdefault(pref.ref, []).append((pref, binary_info)) + @property + def ordered_recipes_by_name(self): + ret = OrderedDict() + for ref, prefs in self.recipes.items(): + ret.setdefault(ref.name, OrderedDict()).setdefault(ref, prefs) + return ret + def serialize(self): - ret = {} + ret = OrderedDict() for ref, prefs in self.recipes.items(): - pref_ret = {} + pref_ret = OrderedDict() if prefs: for pref, binary_info in prefs: pref_ret[pref.repr_notime()] = binary_info diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index f8ed0fb6bdd..e78afdfccc8 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -110,18 +110,24 @@ def select(self, pattern, package_query=None, remote=None, search_mode=None): rrevs = self.conan_api.list.recipe_revisions(r, remote) rrevs = pattern.filter_rrevs(rrevs) select_bundle.add_refs(rrevs) + # Show only the latest recipe revision or all of them if search_mode in (ListPatternMode.SHOW_ALL_RREVS, ListPatternMode.SHOW_LATEST_RREV): continue for rrev in rrevs: - packages = self.conan_api.list.packages_configurations(rrev, remote) - if package_query is not None: - packages = self.conan_api.list.filter_packages_configurations(packages, - package_query) - prefs = packages.keys() - if pattern.package_id is not None: - prefs = pattern.filter_prefs(prefs) + packages = {} + prefs = [] + if pattern.package_id and "*" not in pattern.package_id: + prefs.append(PkgReference(rrev, package_id=pattern.package_id)) + else: + packages = self.conan_api.list.packages_configurations(rrev, remote) + if package_query is not None: + packages = self.conan_api.list.filter_packages_configurations(packages, + package_query) + prefs = packages.keys() + if pattern.package_id is not None: + prefs = pattern.filter_prefs(prefs) # Show all the package IDs and their configurations if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 256fec52fb4..643e8ef3256 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -20,57 +20,57 @@ def print_list_text(results): info = results["results"] search_mode = results["search_mode"] - for remote, refs in info.items(): + indentation = " " + + for remote, info_per_ref_name in info.items(): cli_out_write(f"{remote}:", fg=remote_color) - if refs.get("error"): - cli_out_write(f" ERROR: {refs.get('error')}", fg=error_color) + if info_per_ref_name.get("error"): + cli_out_write(f" ERROR: {info_per_ref_name.get('error')}", fg=error_color) continue - if not refs: + if not info_per_ref_name: cli_out_write(f" There are no matching recipe references", fg=recipe_color) continue - showed_ref_names = [] - indentation = " " - for ref, prefs in refs.items(): - ref_name = ref.name - if ref_name not in showed_ref_names: - showed_ref_names.append(ref_name) - cli_out_write(f"{indentation}{ref_name}", fg=recipe_name_color) - cli_out_write(f"{indentation * 2}{ref.repr_humantime() if ref.timestamp else ref}", - fg=recipe_color) - if prefs: - for pref, binary_info in prefs: - pref_date = f" ({timestamp_to_str(pref.timestamp)})" if pref.timestamp else "" - if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: - cli_out_write(f"{indentation * 3}PID: {pref.package_id}{pref_date}", - fg=reference_color) - if not binary_info: - cli_out_write(f"{indentation * 4}Empty package information", + for ref_name, refs in info_per_ref_name.items(): + cli_out_write(f"{indentation}{ref_name}", fg=recipe_name_color) + for ref, prefs in refs.items(): + cli_out_write(f"{indentation * 2}{ref.repr_humantime() if ref.timestamp else ref}", + fg=recipe_color) + if prefs: + for pref, binary_info in prefs: + pref_date = f" ({timestamp_to_str(pref.timestamp)})" if pref.timestamp \ + else "" + if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: + cli_out_write(f"{indentation * 3}PID: {pref.package_id}{pref_date}", + fg=reference_color) + if not binary_info: + cli_out_write(f"{indentation * 4}Empty package information", + fg=field_color) + continue + elif search_mode in (ListPatternMode.SHOW_ALL_PREVS, + ListPatternMode.SHOW_LATEST_PREV): + cli_out_write(f"{indentation * 3}PID: {pref.package_id}", + fg=reference_color) + cli_out_write(f"{indentation * 4}PREV: {pref.revision}{pref_date}", fg=field_color) continue - elif search_mode in (ListPatternMode.SHOW_ALL_PREVS, - ListPatternMode.SHOW_LATEST_PREV): - cli_out_write(f"{indentation * 3}PID: {pref.package_id}", fg=reference_color) - cli_out_write(f"{indentation * 4}PREV: {pref.revision}{pref_date}", - fg=field_color) - continue - for item, contents in binary_info.items(): - if not contents: - continue - cli_out_write(f"{indentation * 4}{item}:", fg=field_color) - if isinstance(contents, dict): - for k, v in contents.items(): - cli_out_write(f"{indentation * 5}{k}={v}", fg=value_color) - else: - for c in contents: - cli_out_write(f"{indentation * 5}{c}", fg=value_color) - elif search_mode in (ListPatternMode.SHOW_PACKAGE_IDS, - ListPatternMode.SHOW_ALL_PREVS, - ListPatternMode.SHOW_LATEST_PREV): - cli_out_write(f"{indentation * 3}There are no packages for this revision.", - fg=field_color) + for item, contents in binary_info.items(): + if not contents: + continue + cli_out_write(f"{indentation * 4}{item}:", fg=field_color) + if isinstance(contents, dict): + for k, v in contents.items(): + cli_out_write(f"{indentation * 5}{k}={v}", fg=value_color) + else: + for c in contents: + cli_out_write(f"{indentation * 5}{c}", fg=value_color) + elif search_mode in (ListPatternMode.SHOW_PACKAGE_IDS, + ListPatternMode.SHOW_ALL_PREVS, + ListPatternMode.SHOW_LATEST_PREV): + cli_out_write(f"{indentation * 3}There are no packages for this revision.", + fg=field_color) def print_list_json(data): @@ -114,7 +114,7 @@ def list(conan_api: ConanAPI, parser, *args): results[name] = {"error": str(e)} else: results[name] = list_bundle.serialize() if args.format in ("json", "html") \ - else list_bundle.recipes + else list_bundle.ordered_recipes_by_name return { "results": results, "search_mode": ref_pattern.mode, diff --git a/conan/cli/commands/search.py b/conan/cli/commands/search.py index f3c378438d9..a6bb9fa9609 100644 --- a/conan/cli/commands/search.py +++ b/conan/cli/commands/search.py @@ -36,7 +36,7 @@ def search(conan_api: ConanAPI, parser, *args): results[remote.name] = {"error": str(e)} else: results[remote.name] = list_bundle.serialize() if args.format == "json" \ - else list_bundle.recipes + else list_bundle.ordered_recipes_by_name return { "results": results, "search_mode": search_mode, diff --git a/conans/test/integration/command_v2/list_test.py b/conans/test/integration/command_v2/list_test.py index 6b345e03239..32499ba9a88 100644 --- a/conans/test/integration/command_v2/list_test.py +++ b/conans/test/integration/command_v2/list_test.py @@ -47,6 +47,19 @@ def _upload_full_recipe(self, remote, ref): f"--user={ref.user} --channel={ref.channel}") self.client.run("upload --force -r {} {}".format(remote, ref)) + def _upload_full_recipe_without_conaninfo(self, remote, ref): + self.client.save({"conanfile.py": GenConanfile("pkg", "0.1").with_package_file("file.h", + "0.1")}) + self.client.run("create . --user=user --channel=channel") + self.client.run("upload --force -r {} {}".format(remote, "pkg/0.1@user/channel")) + + self.client.save({'conanfile.py': GenConanfile() + }) + self.client.run(f"create . --name={ref.name} --version={ref.version} " + f"--user={ref.user} --channel={ref.channel}") + self.client.run("upload --force -r {} {}".format(remote, ref)) + + @staticmethod def _get_fake_recipe_refence(recipe_name): return f"{recipe_name}#fca0383e6a43348f7989f11ab8f0a92d" @@ -332,6 +345,99 @@ def test_search_package_ids_from_latest_rrev_in_all_remotes_and_cache(self): """) assert bool(re.match(expected_output, output, re.MULTILINE)) + def test_search_all_revisions_and_package_revisions(self): + """Checking if RREVs and PREVs are shown correctly""" + remote1 = "remote1" + remote2 = "remote2" + + self._add_remote(remote1) + self._upload_full_recipe_without_conaninfo(remote1, + RecipeReference(name="test_recipe", version="1.0", + user="user", channel="channel")) + self._add_remote(remote2) + self._upload_full_recipe_without_conaninfo(remote2, + RecipeReference(name="test_recipe", version="2.1", + user="user", channel="channel")) + self.client.run(f'list *#* -r="*" -c') + output = str(self.client.out) + expected_output = textwrap.dedent("""\ + Local Cache: + test_recipe + test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + pkg + pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* + remote1: + pkg + pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* + test_recipe + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + remote2: + pkg + pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* + test_recipe + test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + """) + assert bool(re.match(expected_output, output, re.MULTILINE)) + self.client.run(f'list test_recipe/*:*#* -r="*" -c') + output = str(self.client.out) + expected_output = textwrap.dedent("""\ + Local Cache: + test_recipe + test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* + remote1: + test_recipe + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* + remote2: + test_recipe + test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* + """) + assert bool(re.match(expected_output, output, re.MULTILINE)) + + def test_search_all_revisions_given_a_package_id(self): + """ + Checking if PREVs are shown correctly given a PkgID and even though that package has no + configuration at all. + """ + remote1 = "remote1" + self._add_remote(remote1) + self._upload_full_recipe_without_conaninfo(remote1, + RecipeReference(name="test_recipe", version="1.0", + user="user", channel="channel")) + self.client.run(f'list *:* -r=remote1') + output = str(self.client.out) + expected_output = textwrap.dedent("""\ + remote1: + pkg + pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + Empty package information + test_recipe + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + Empty package information + """) + assert bool(re.match(expected_output, output, re.MULTILINE)) + self.client.run(f'list test_recipe/*:da39a3ee5e6b4b0d3255bfef95601890afd80709#* -r=remote1') + output = str(self.client.out) + expected_output = textwrap.dedent("""\ + remote1: + test_recipe + test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* + PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 + PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* + """) + assert bool(re.match(expected_output, output, re.MULTILINE)) + def test_list_package_query_options(self): self.client.save({"conanfile.py": GenConanfile("pkg", "0.1") .with_package_file("file.h", "0.1") From b560c75b4d6bdf784d6835a553588eb67965a304 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 29 Dec 2022 14:36:20 +0100 Subject: [PATCH 20/62] Test/conan info not check server (#12807) * test that conan info will not check server if pkgs in cache * more checks --- .../integration/command/info/info_test.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/conans/test/integration/command/info/info_test.py b/conans/test/integration/command/info/info_test.py index 2c5faafa8ba..a5992a4c6ea 100644 --- a/conans/test/integration/command/info/info_test.py +++ b/conans/test/integration/command/info/info_test.py @@ -747,3 +747,26 @@ def build_requirements(self): client.run("info . " + args) assert "AttributeError: 'HelloConan' object has no attribute 'tested_reference_str'"\ not in client.out + + +def test_info_not_hit_server(): + c = TestClient(default_server_user=True) + c.save({"pkg/conanfile.py": GenConanfile("pkg", "0.1"), + "consumer/conanfile.py": GenConanfile("consumer", "0.1").with_require("pkg/0.1")}) + c.run("create pkg") + c.run("create consumer") + c.run("upload * -r=default --all -c") + c.run("remove * -f") + c.run("install consumer/0.1@") + assert "Downloaded" in c.out + # break the server to make sure it is not being contacted at all + c.servers["default"] = None + c.run("info consumer/0.1@") + assert "Downloaded" not in c.out + c.run("remove pkg* -f") + c.run("info consumer/0.1@", assert_error=True) + assert "ERROR: 'NoneType' object " in c.out + c.run("remote disable *") + c.run("info consumer/0.1@", assert_error=True) + assert "ERROR: 'NoneType' object " not in c.out + assert "ERROR: No remote defined" in c.out From 7947a23c0638aad19899f5d4ce87713695fc1526 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 30 Dec 2022 12:00:22 +0100 Subject: [PATCH 21/62] [develop2] allow ``--no-remotes`` to force temporal disabling of remotes (#12808) --- conan/cli/args.py | 7 +++++-- conan/cli/commands/build.py | 2 +- conan/cli/commands/create.py | 2 +- conan/cli/commands/export.py | 9 ++++++--- conan/cli/commands/graph.py | 4 ++-- conan/cli/commands/install.py | 2 +- conan/cli/commands/lock.py | 2 +- conan/cli/commands/test.py | 2 +- .../test/integration/command/install/install_test.py | 11 +++++++++++ 9 files changed, 29 insertions(+), 12 deletions(-) diff --git a/conan/cli/args.py b/conan/cli/args.py index 9533109f92b..b6651a0d055 100644 --- a/conan/cli/args.py +++ b/conan/cli/args.py @@ -38,8 +38,11 @@ def _add_common_install_arguments(parser, build_help, update_help=None): if build_help: parser.add_argument("-b", "--build", action="append", help=build_help) - parser.add_argument("-r", "--remote", action="append", default=None, - help='Look in the specified remote or remotes server') + group = parser.add_mutually_exclusive_group() + group.add_argument("-r", "--remote", action="append", default=None, + help='Look in the specified remote or remotes server') + group.add_argument("-nr", "--no-remote", action="store_true", + help='Do not use remote, resolve exclusively in the cache') if not update_help: update_help = ("Will check the remote and in case a newer version and/or revision of " diff --git a/conan/cli/commands/build.py b/conan/cli/commands/build.py index dbe84efb1c3..923cbdf6f2f 100644 --- a/conan/cli/commands/build.py +++ b/conan/cli/commands/build.py @@ -29,7 +29,7 @@ def build(conan_api, parser, *args): cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=True) folder = os.path.dirname(path) - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index 2ba5c58afbf..bb46285d1d0 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -44,7 +44,7 @@ def create(conan_api, parser, *args): conanfile_path=path, cwd=cwd, partial=args.lockfile_partial) - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) out = ConanOutput() diff --git a/conan/cli/commands/export.py b/conan/cli/commands/export.py index 0e03b9d8edf..4fbb1634968 100644 --- a/conan/cli/commands/export.py +++ b/conan/cli/commands/export.py @@ -21,8 +21,11 @@ def export(conan_api, parser, *args): Export recipe to the Conan package cache """ common_args_export(parser) - parser.add_argument("-r", "--remote", action="append", default=None, - help='Look in the specified remote or remotes server') + group = parser.add_mutually_exclusive_group() + group.add_argument("-r", "--remote", action="append", default=None, + help='Look in the specified remote or remotes server') + group.add_argument("-nr", "--no-remote", action="store_true", + help='Do not use remote, resolve exclusively in the cache') parser.add_argument("-l", "--lockfile", action=OnceArgument, help="Path to a lockfile.") parser.add_argument("--lockfile-out", action=OnceArgument, @@ -35,7 +38,7 @@ def export(conan_api, parser, *args): cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=True) - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 36312f7484c..5da4c62a36f 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -50,7 +50,7 @@ def graph_build_order(conan_api, parser, subparser, *args): path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None # Basic collaborators, remotes, lockfile, profiles - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, @@ -132,7 +132,7 @@ def graph_info(conan_api, parser, subparser, *args): path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None # Basic collaborators, remotes, lockfile, profiles - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, diff --git a/conan/cli/commands/install.py b/conan/cli/commands/install.py index 5e005106371..286d015601e 100644 --- a/conan/cli/commands/install.py +++ b/conan/cli/commands/install.py @@ -61,7 +61,7 @@ def install(conan_api, parser, *args): output_folder = None # Basic collaborators, remotes, lockfile, profiles - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, diff --git a/conan/cli/commands/lock.py b/conan/cli/commands/lock.py index 14b98caf678..3a9c73501d1 100644 --- a/conan/cli/commands/lock.py +++ b/conan/cli/commands/lock.py @@ -32,7 +32,7 @@ def lock_create(conan_api, parser, subparser, *args): cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] lockfile = conan_api.lockfile.get_lockfile(lockfile=args.lockfile, conanfile_path=path, cwd=cwd, partial=True) profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) diff --git a/conan/cli/commands/test.py b/conan/cli/commands/test.py index ca4018a5f22..8dea6e20075 100644 --- a/conan/cli/commands/test.py +++ b/conan/cli/commands/test.py @@ -28,7 +28,7 @@ def test(conan_api, parser, *args): conanfile_path=path, cwd=cwd, partial=args.lockfile_partial) - remotes = conan_api.remotes.list(args.remote) + remotes = conan_api.remotes.list(args.remote) if not args.no_remote else [] profile_host, profile_build = conan_api.profiles.get_profiles_from_args(args) out = ConanOutput() diff --git a/conans/test/integration/command/install/install_test.py b/conans/test/integration/command/install/install_test.py index 3f9ac7d6d21..303485cb2ca 100644 --- a/conans/test/integration/command/install/install_test.py +++ b/conans/test/integration/command/install/install_test.py @@ -298,6 +298,17 @@ def test_install_disabled_remote(client): assert "ERROR: Remote 'default' can't be found or is disabled" in client.out +def test_install_no_remotes(client): + client.save({"conanfile.py": GenConanfile("pkg", "0.1")}) + client.run("create .") + client.run("upload * --confirm -r default") + client.run("remove * -c") + client.run("install --requires=pkg/0.1 -nr", assert_error=True) + assert "ERROR: Package 'pkg/0.1' not resolved: No remote defined" in client.out + client.run("install --requires=pkg/0.1") # this works without issue + client.run("install --requires=pkg/0.1 -nr") # and now this too, pkg in cache + + def test_install_skip_disabled_remote(): client = TestClient(servers=OrderedDict({"default": TestServer(), "server2": TestServer(), From 8def1526dda7fee32dcda1a370bdf13771cccd4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Mon, 2 Jan 2023 10:30:01 +0100 Subject: [PATCH 22/62] Add barebones template to `conan new` (#12802) * Add barebones template to conan new * Properly rename to barebones * Improvements to template naming * Improve help command * Move rendering to jinja, keep logic in py * Fix test * Revert unwanted file * Add help text to some commands * Update conans/test/integration/command/new_test.py Co-authored-by: James * Address PR review * Update conans/test/integration/command/new_test.py Co-authored-by: James * Address PR review * Fix indent Co-authored-by: James --- conan/api/subapi/new.py | 6 ++- conan/cli/commands/new.py | 21 +++++--- conan/internal/api/new/basic.py | 54 +++++++++++++++++++++ conans/test/integration/command/new_test.py | 11 +++++ 4 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 conan/internal/api/new/basic.py diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index 88bde617c07..e102798b756 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -17,6 +17,7 @@ def __init__(self, conan_api): @api_method def get_builtin_template(self, template_name): + from conan.internal.api.new.basic import basic_file from conan.internal.api.new.alias_new import alias_file from conan.internal.api.new.cmake_exe import cmake_exe_files from conan.internal.api.new.cmake_lib import cmake_lib_files @@ -28,7 +29,8 @@ def get_builtin_template(self, template_name): from conan.internal.api.new.bazel_exe import bazel_exe_files from conan.internal.api.new.autotools_lib import autotools_lib_files from conan.internal.api.new.autoools_exe import autotools_exe_files - new_templates = {"cmake_lib": cmake_lib_files, + new_templates = {"basic": basic_file, + "cmake_lib": cmake_lib_files, "cmake_exe": cmake_exe_files, "meson_lib": meson_lib_files, "meson_exe": meson_exe_files, @@ -82,12 +84,14 @@ def _read_files(self, template_folder): return template_files, non_template_files + @staticmethod def render(template_files, definitions): result = {} name = definitions.get("name", "Pkg") definitions["conan_version"] = __version__ definitions["package_name"] = name.replace("-", "_").replace("+", "_").replace(".", "_") + definitions["as_iterable"] = lambda x: [x] if isinstance(x, str) else x for k, v in template_files.items(): k = Template(k, keep_trailing_newline=True, undefined=StrictUndefined).render(**definitions) v = Template(v, keep_trailing_newline=True, undefined=StrictUndefined).render(**definitions) diff --git a/conan/cli/commands/new.py b/conan/cli/commands/new.py index e512f767d44..e8c2011b975 100644 --- a/conan/cli/commands/new.py +++ b/conan/cli/commands/new.py @@ -12,18 +12,20 @@ @conan_command(group="Creator") def new(conan_api, parser, *args): """ - Create a new recipe (with conanfile.py and other files) from a predefined template + Create a new recipe (with conanfile.py and other files) from either a predefined or a user-defined template """ - parser.add_argument("template", help="Template name, built-in predefined one or user one. " - "You can use built-in templates: cmake_lib, cmake_exe, " + parser.add_argument("template", help="Template name, " + "either a predefined built-in or a user-provided one. " + "Available built-in templates: basic, cmake_lib, cmake_exe, " "meson_lib, meson_exe, msbuild_lib, msbuild_exe, bazel_lib, bazel_exe, " "autotools_lib, autotools_exe. " "E.g. 'conan new cmake_lib -d name=hello -d version=0.1'. " - "You can define your own templates too." + "You can define your own templates too by inputting an absolute path " + "as your template, or a path relative to your conan home folder." ) parser.add_argument("-d", "--define", action="append", help="Define a template argument as key=value") - parser.add_argument("-f", "--force", action='store_true', help="Overwrite file if exists") + parser.add_argument("-f", "--force", action='store_true', help="Overwrite file if it already exists") args = parser.parse_args(*args) # Manually parsing the remainder @@ -34,7 +36,14 @@ def new(conan_api, parser, *args): except ValueError: raise ConanException(f"Template definitions must be 'key=value', received {u}") k = k.replace("-", "") # Remove possible "--name=value" - definitions[k] = v + # For variables that only show up once, no need for list to keep compatible behaviour + if k in definitions: + if isinstance(definitions[k], list): + definitions[k].append(v) + else: + definitions[k] = [definitions[k], v] + else: + definitions[k] = v files = conan_api.new.get_template(args.template) # First priority: user folder if not files: # then, try the templates in the Conan home diff --git a/conan/internal/api/new/basic.py b/conan/internal/api/new/basic.py new file mode 100644 index 00000000000..1bdc7c76815 --- /dev/null +++ b/conan/internal/api/new/basic.py @@ -0,0 +1,54 @@ +def inject_get_or_else(variable, default): + return variable + ' = "{% if ' + variable + " is defined %}{{ " + variable + " }}{% else %}" + default + '{% endif %}"' + +_conanfile_header = f'''\ + {inject_get_or_else("name", "pkg")} + {inject_get_or_else("version", "1.0")} + {inject_get_or_else("description", "A basic recipe")} + {inject_get_or_else("license", "")} + {inject_get_or_else("homepage", "")} +''' + +_conanfile = '''\ +from conan import ConanFile + +class BasicConanfile(ConanFile): +''' + _conanfile_header + '''\ + + # The requirements method allows you to define the dependencies of your recipe + def requirements(self): + # Each call to self.requires() will add the corresponding requirement + # to the current list of requirements + {% if requires is defined -%} + {% for require in as_iterable(requires) -%} + self.requires("{{ require }}") + {% endfor %} + {% else -%} + # Uncommenting this line will add the zlib/1.2.13 dependency to your project + # self.requires("zlib/1.2.13") + pass + {%- endif %} + + # The purpose of generate() is to prepare the build, generating the necessary files, such as + # Files containing information to locate the dependencies, environment activation scripts, + # and specific build system files among others + def generate(self): + pass + + # This method is used to build the source code of the recipe using the desired commands. + def build(self): + # You can use your command line tools to invoke your build system + # or any of the build helpers provided with Conan in conan.tools + # self.run("g++ ...") + pass + + # The actual creation of the package, once it's built, is done in the package() method. + # Using the copy() method from tools.files, artifacts are copied + # from the build folder to the package folder + def package(self): + # copy(self, "*.h", self.source_folder, join(self.package_folder, "include"), keep_path=False) + pass +''' + + +basic_file = {"conanfile.py": _conanfile} diff --git a/conans/test/integration/command/new_test.py b/conans/test/integration/command/new_test.py index 27015782249..32a46b8eb76 100644 --- a/conans/test/integration/command/new_test.py +++ b/conans/test/integration/command/new_test.py @@ -45,6 +45,17 @@ def test_new_missing_definitions(self): client.run("new cmake_lib -d version=myversion", assert_error=True) assert "Missing definitions for the template. Required definitions are: 'name', 'version'" in client.out + def test_new_basic_template(self): + tc = TestClient() + tc.run("new basic") + assert '# self.requires("zlib/1.2.13")' in tc.load("conanfile.py") + + tc.run("new basic -d name=mygame -d requires=math/1.0 -d requires=ai/1.0 -f") + conanfile = tc.load("conanfile.py") + assert 'self.requires("math/1.0")' in conanfile + assert 'self.requires("ai/1.0")' in conanfile + assert 'name = "mygame"' in conanfile + class TestNewCommandUserTemplate: From 2e9f6b05674f8f57de5ff930a94d9145e8c0c5ac Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 2 Jan 2023 12:30:26 +0100 Subject: [PATCH 23/62] review --- .codeclimate.yml | 23 ----------------------- README.rst | 19 ++++++++----------- 2 files changed, 8 insertions(+), 34 deletions(-) delete mode 100644 .codeclimate.yml diff --git a/.codeclimate.yml b/.codeclimate.yml deleted file mode 100644 index 03b9b11106f..00000000000 --- a/.codeclimate.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: "2" -plugins: - pep8: - enabled: true -checks: - file-lines: - config: - threshold: 5000 - argument-count: - config: - threshold: 10 - complex-logic: - config: - threshold: 100 - method-lines: - config: - threshold: 100 - method-count: - config: - threshold: 50 - method-complexity: - config: - threshold: 20 diff --git a/README.rst b/README.rst index a8ac30db90b..ca6229e0afa 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ Decentralized, open-source (MIT), C/C++ package manager. - Homepage: https://conan.io/ - Github: https://github.com/conan-io/conan - Docs: https://docs.conan.io/en/latest/ -- Slack: https://cpplang.slack.com/?id=conan (#conan channel) +- Slack: https://cpplang.slack.com (#conan channel) - Twitter: https://twitter.com/conan_io @@ -28,15 +28,15 @@ Conan is a package manager for C and C++ developers: - Stable. Used in production by many companies, since 1.0 there is a commitment not to break package recipes and documented behavior. +This is the **developer/maintainer** documentation. For user documentation, go to https://docs.conan.io -+-------------------------+-------------------------+ -| **develop** | **Code Climate** | -+=========================+=========================+ -| |Build Status Develop| | |Develop climate| | -+-------------------------+-------------------------+ ++-------------------------+ +| **develop2** | ++=========================+ +| |Build Status Develop| | ++-------------------------+ -This is the **developer/maintainer** documentation. For user documentation, go to https://docs.conan.io Setup ===== @@ -165,10 +165,7 @@ License `MIT LICENSE <./LICENSE.md>`__ .. |Build Status Develop| image:: https://ci.conan.io/buildStatus/icon?job=ConanTestSuite/develop - :target: https://ci.conan.io/job/ConanTestSuite/job/develop/ - -.. |Develop climate| image:: https://api.codeclimate.com/v1/badges/081b53e570d5220b34e4/maintainability.svg - :target: https://codeclimate.com/github/conan-io/conan/maintainability + :target: https://ci.conan.io/blue/organizations/jenkins/ConanTestSuitev2/activity .. |Logo| image:: https://conan.io/img/jfrog_conan_logo.png From 28085bb2d79d7b997c5b4986a201cabd4f237802 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 2 Jan 2023 12:55:19 +0100 Subject: [PATCH 24/62] [develop2] removing dead code related to rest, remote, downloads (#12822) --- conan/tools/files/files.py | 3 ++ conans/client/cmd/uploader.py | 16 +------ conans/client/downloaders/file_downloader.py | 44 +++++++------------ conans/client/remote_manager.py | 8 +--- conans/client/rest/rest_client.py | 7 +-- conans/client/rest/rest_client_common.py | 11 +---- conans/client/rest/rest_client_v2.py | 13 +----- conans/server/store/server_store.py | 6 --- .../test/integration/remote/rest_api_test.py | 2 +- conans/util/tracer.py | 11 ----- 10 files changed, 26 insertions(+), 95 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 946ad08296e..ae239e61ccc 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -228,6 +228,8 @@ def download(conanfile, url, filename, verify=True, retry=None, retry_wait=None, if download_cache and not os.path.isabs(download_cache): raise ConanException("core.download:download_cache must be an absolute path") + filename = os.path.abspath(filename) + def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download if file_url.startswith("file:"): @@ -235,6 +237,7 @@ def _download_file(file_url): sha1=sha1, sha256=sha256) else: downloader = CachingFileDownloader(requester, download_cache=download_cache) + os.makedirs(os.path.dirname(filename), exist_ok=True) # filename in subfolder must exist downloader.download(url=file_url, file_path=filename, auth=auth, overwrite=overwrite, verify_ssl=verify, retry=retry, retry_wait=retry_wait, headers=headers, md5=md5, sha1=sha1, sha256=sha256) diff --git a/conans/client/cmd/uploader.py b/conans/client/cmd/uploader.py index 0e39110bbbb..aecb108fac0 100644 --- a/conans/client/cmd/uploader.py +++ b/conans/client/cmd/uploader.py @@ -282,26 +282,12 @@ def upload(self, upload_data, remote): if package.upload: self.upload_package(package, remote) - def _recipe_files_to_upload(self, ref, files, remote, force): - if not force: - return files, set() - # only check difference if it is a force upload - remote_snapshot = self._app.remote_manager.get_recipe_snapshot(ref, remote) - if not remote_snapshot: - return files, set() - - deleted = set(remote_snapshot).difference(files) - return files, deleted - def upload_recipe(self, ref, bundle, remote): self._output.info(f"Uploading recipe '{ref.repr_notime()}'") t1 = time.time() cache_files = bundle.files - force = bundle.force - files_to_upload, deleted = self._recipe_files_to_upload(ref, cache_files, remote, force) - upload_ref = ref - self._app.remote_manager.upload_recipe(upload_ref, files_to_upload, deleted, remote) + self._app.remote_manager.upload_recipe(ref, cache_files, remote) duration = time.time() - t1 log_recipe_upload(ref, duration, cache_files, remote.name) diff --git a/conans/client/downloaders/file_downloader.py b/conans/client/downloaders/file_downloader.py index d4c976d3149..0a76a0d2802 100644 --- a/conans/client/downloaders/file_downloader.py +++ b/conans/client/downloaders/file_downloader.py @@ -7,7 +7,6 @@ from conans.client.rest import response_to_str from conans.errors import ConanException, NotFoundException, AuthenticationException, \ ForbiddenException, ConanConnectionError, RequestErrorException -from conans.util.files import mkdir from conans.util.sha import check_with_algorithm_sum from conans.util.tracer import log_download @@ -20,9 +19,10 @@ def __init__(self, requester): def download(self, url, file_path, retry=2, retry_wait=0, verify_ssl=True, auth=None, overwrite=False, headers=None, md5=None, sha1=None, sha256=None): - - if not os.path.isabs(file_path): - file_path = os.path.abspath(file_path) + """ in order to make the download concurrent, the folder for file_path MUST exist + """ + assert file_path, "Conan 2.0 always download files to disk, not to memory" + assert os.path.isabs(file_path), "Target file_path must be absolute" if os.path.exists(file_path): if overwrite: @@ -65,7 +65,7 @@ def _check_checksum(file_path, md5, sha1, sha256): def _download_file(self, url, auth, headers, file_path, verify_ssl, try_resume=False): t1 = time.time() - if try_resume and file_path and os.path.exists(file_path): + if try_resume and os.path.exists(file_path): range_start = os.path.getsize(file_path) headers = headers.copy() if headers else {} headers["range"] = "bytes={}-".format(range_start) @@ -90,24 +90,6 @@ def _download_file(self, url, auth, headers, file_path, verify_ssl, try_resume=F raise AuthenticationException() raise ConanException("Error %d downloading file %s" % (response.status_code, url)) - def read_response(chunk_size, path=None): - ret = None - downloaded_size = range_start - if path: - mkdir(os.path.dirname(path)) - mode = "ab" if range_start else "wb" - with open(path, mode) as file_handler: - for chunk in response.iter_content(chunk_size): - file_handler.write(chunk) - downloaded_size += len(chunk) - else: - ret_data = bytearray() - for chunk in response.iter_content(chunk_size): - ret_data.extend(chunk) - downloaded_size += len(chunk) - ret = bytes(ret_data) - return ret, downloaded_size - def get_total_length(): if range_start: content_range = response.headers.get("Content-Range", "") @@ -127,23 +109,27 @@ def get_total_length(): if description: self._output.info(description) - chunksize = 1024 if not file_path else 1024 * 100 - written_chunks, total_downloaded_size = read_response(chunksize, file_path) + chunk_size = 1024 * 100 + total_downloaded_size = range_start + mode = "ab" if range_start else "wb" + with open(file_path, mode) as file_handler: + for chunk in response.iter_content(chunk_size): + file_handler.write(chunk) + total_downloaded_size += len(chunk) + gzip = (response.headers.get("content-encoding") == "gzip") response.close() # it seems that if gzip we don't know the size, cannot resume and shouldn't raise if total_downloaded_size != total_length and not gzip: - if (file_path and total_length > total_downloaded_size > range_start + if (total_length > total_downloaded_size > range_start and response.headers.get("Accept-Ranges") == "bytes"): - written_chunks = self._download_file(url, auth, headers, file_path, verify_ssl, - try_resume=True) + self._download_file(url, auth, headers, file_path, verify_ssl, try_resume=True) else: raise ConanException("Transfer interrupted before complete: %s < %s" % (total_downloaded_size, total_length)) duration = time.time() - t1 log_download(url, duration) - return written_chunks except Exception as e: # If this part failed, it means problems with the connection to server diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 14bbb1d2d1a..49fa5326f92 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -35,14 +35,10 @@ def __init__(self, cache, auth_manager, hook_manager): def check_credentials(self, remote): self._call_remote(remote, "check_credentials") - def get_recipe_snapshot(self, ref, remote): - assert ref.revision, "get_recipe_snapshot requires revision" - return self._call_remote(remote, "get_recipe_snapshot", ref) - - def upload_recipe(self, ref, files_to_upload, deleted, remote): + def upload_recipe(self, ref, files_to_upload, remote): assert isinstance(ref, RecipeReference) assert ref.revision, "upload_recipe requires RREV" - self._call_remote(remote, "upload_recipe", ref, files_to_upload, deleted) + self._call_remote(remote, "upload_recipe", ref, files_to_upload) def upload_package(self, pref, files_to_upload, remote): assert pref.ref.revision, "upload_package requires RREV" diff --git a/conans/client/rest/rest_client.py b/conans/client/rest/rest_client.py index 544a3a04f82..ebfe2505503 100644 --- a/conans/client/rest/rest_client.py +++ b/conans/client/rest/rest_client.py @@ -63,17 +63,14 @@ def _get_api(self): def get_recipe(self, ref, dest_folder): return self._get_api().get_recipe(ref, dest_folder) - def get_recipe_snapshot(self, ref): - return self._get_api().get_recipe_snapshot(ref) - def get_recipe_sources(self, ref, dest_folder): return self._get_api().get_recipe_sources(ref, dest_folder) def get_package(self, pref, dest_folder): return self._get_api().get_package(pref, dest_folder) - def upload_recipe(self, ref, files_to_upload, deleted): - return self._get_api().upload_recipe(ref, files_to_upload, deleted) + def upload_recipe(self, ref, files_to_upload): + return self._get_api().upload_recipe(ref, files_to_upload) def upload_package(self, pref, files_to_upload): return self._get_api().upload_package(pref, files_to_upload) diff --git a/conans/client/rest/rest_client_common.py b/conans/client/rest/rest_client_common.py index 6b52f52bada..518b67ff554 100644 --- a/conans/client/rest/rest_client_common.py +++ b/conans/client/rest/rest_client_common.py @@ -203,18 +203,9 @@ def get_json(self, url, data=None, headers=None): raise ConanException("Unexpected server response %s" % result) return result - def upload_recipe(self, ref, files_to_upload, deleted): + def upload_recipe(self, ref, files_to_upload): if files_to_upload: self._upload_recipe(ref, files_to_upload) - if deleted: - self._remove_recipe_files(ref, deleted) - - def get_recipe_snapshot(self, ref): - # this method is used only for UPLOADING, then it requires the credentials - # Check of credentials is done in the uploader - url = self.router.recipe_snapshot(ref) - snap = self._get_snapshot(url) - return snap def upload_package(self, pref, files_to_upload): self._upload_package(pref, files_to_upload) diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index de3367b637b..0f4e04bcc62 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -34,14 +34,6 @@ def _get_file_list_json(self, url): data["files"] = list(data["files"].keys()) return data - def _get_snapshot(self, url): - try: - data = self._get_file_list_json(url) - files_list = [os.path.normpath(filename) for filename in data["files"]] - except NotFoundException: - files_list = [] - return files_list - def get_recipe(self, ref, dest_folder): url = self.router.recipe_snapshot(ref) data = self._get_file_list_json(url) @@ -153,13 +145,10 @@ def _download_and_save_files(self, urls, dest_folder, files): for filename in sorted(files, reverse=True): resource_url = urls[filename] abs_path = os.path.join(dest_folder, filename) + os.makedirs(os.path.dirname(abs_path), exist_ok=True) # filename in subfolder must exist downloader.download(url=resource_url, file_path=abs_path, auth=self.auth, verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait) - def _remove_recipe_files(self, ref, files): - # V2 === revisions, do not remove files, it will create a new revision if the files changed - return - def remove_all_packages(self, ref): """ Remove all packages from the specified reference""" self.check_credentials() diff --git a/conans/server/store/server_store.py b/conans/server/store/server_store.py index 1c35d830147..ba3697c5220 100644 --- a/conans/server/store/server_store.py +++ b/conans/server/store/server_store.py @@ -138,12 +138,6 @@ def remove_all_packages(self, ref): packages_folder = self.packages(ref) self._storage_adapter.delete_folder(packages_folder) - def remove_recipe_files(self, ref, files): - subpath = self.export(ref) - for filepath in files: - path = join(subpath, filepath) - self._storage_adapter.delete_file(path) - def remove_package_files(self, pref, files): subpath = self.package(pref) for filepath in files: diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index 490f5ade8a5..fbe8abff11c 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -241,4 +241,4 @@ class MyConan(ConanFile): abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST) conan_digest.save(tmp_dir) - self.api.upload_recipe(ref, abs_paths, None) + self.api.upload_recipe(ref, abs_paths) diff --git a/conans/util/tracer.py b/conans/util/tracer.py index 71242e407b3..4ead57b22b6 100644 --- a/conans/util/tracer.py +++ b/conans/util/tracer.py @@ -118,17 +118,6 @@ def log_conan_api_call(name, kwargs): ConanOutput().trace(data) -def log_command(name, kwargs): - parameters = copy.copy(kwargs) # Ensure we don't alter any app object like args - data = {"_action": "COMMAND", "name": name, "parameters": parameters} - ConanOutput().trace(data) - - -def log_exception(exc, message): - data = {"_action": "EXCEPTION", "class": str(exc.__class__.__name__), "message": message} - ConanOutput().trace(data) - - def log_download(url, duration): data = {"_action": "DOWNLOAD", "url": url, "duration": duration} ConanOutput().trace(data) From 95b43e53d3d728707f561b46bbf2f21850687fae Mon Sep 17 00:00:00 2001 From: James Date: Tue, 3 Jan 2023 14:17:44 +0100 Subject: [PATCH 25/62] Feature/develop2 restapi parallel download (#12828) * wip * fix tests --- .../internal/cache/conan_reference_layout.py | 1 - conans/client/downloaders/file_downloader.py | 5 ++--- conans/client/graph/proxy.py | 5 ----- conans/client/remote_manager.py | 3 +-- conans/client/rest/rest_client_v2.py | 22 +++++++++++++++---- .../integration/cache/cache2_update_test.py | 1 - .../unittests/tools/microsoft/conanvcvars.bat | 2 -- conans/util/thread.py | 15 +++++++++++++ conans/util/tracer.py | 18 --------------- 9 files changed, 36 insertions(+), 36 deletions(-) delete mode 100644 conans/test/unittests/tools/microsoft/conanvcvars.bat create mode 100644 conans/util/thread.py diff --git a/conan/internal/cache/conan_reference_layout.py b/conan/internal/cache/conan_reference_layout.py index ea44c3cab37..1412368b21c 100644 --- a/conan/internal/cache/conan_reference_layout.py +++ b/conan/internal/cache/conan_reference_layout.py @@ -1,7 +1,6 @@ import os from contextlib import contextmanager -from conans.errors import ConanException from conans.model.manifest import FileTreeManifest from conans.paths import CONANFILE, DATA_YML from conans.util.files import set_dirty, clean_dirty, is_dirty, rmdir diff --git a/conans/client/downloaders/file_downloader.py b/conans/client/downloaders/file_downloader.py index 0a76a0d2802..a2736b00ec1 100644 --- a/conans/client/downloaders/file_downloader.py +++ b/conans/client/downloaders/file_downloader.py @@ -105,9 +105,8 @@ def get_total_length(): try: total_length = get_total_length() action = "Downloading" if range_start == 0 else "Continuing download of" - description = "{} {}".format(action, os.path.basename(file_path)) if file_path else None - if description: - self._output.info(description) + description = "{} {}".format(action, os.path.basename(file_path)) + self._output.info(description) chunk_size = 1024 * 100 total_downloaded_size = range_start diff --git a/conans/client/graph/proxy.py b/conans/client/graph/proxy.py index 277d50ea064..bcd50718e7e 100644 --- a/conans/client/graph/proxy.py +++ b/conans/client/graph/proxy.py @@ -3,7 +3,6 @@ RECIPE_NOT_IN_REMOTE, RECIPE_UPDATED, RECIPE_EDITABLE, RECIPE_INCACHE_DATE_UPDATED, RECIPE_UPDATEABLE) from conans.errors import ConanException, NotFoundException -from conans.util.tracer import log_recipe_got_from_local_cache class ConanProxy(object): @@ -19,10 +18,6 @@ def get_recipe(self, ref, remotes, update, check_update): # with layout.conanfile_write_lock(self._out): result = self._get_recipe(ref, remotes) conanfile_path, status, remote, new_ref = result - - if status not in (RECIPE_DOWNLOADED, RECIPE_UPDATED): - log_recipe_got_from_local_cache(new_ref) - return conanfile_path, status, remote, new_ref # return the remote where the recipe was found or None if the recipe was not found diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index 49fa5326f92..dcd10a973f7 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -20,7 +20,7 @@ # FIXME: Eventually, when all output is done, tracer functions should be moved to the recorder class from conans.util.tracer import (log_package_download, - log_recipe_download, log_recipe_sources_download, + log_recipe_sources_download, log_uncompressed_file) @@ -64,7 +64,6 @@ def get_recipe(self, ref, remote): ref_time = remote_refs[0].timestamp ref.timestamp = ref_time duration = time.time() - t1 - log_recipe_download(ref, duration, remote.name, zipped_files) # filter metadata files # This could be also optimized in the download, avoiding downloading them, for performance zipped_files = {k: v for k, v in zipped_files.items() if not k.startswith(METADATA)} diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index 0f4e04bcc62..a77f47e8ec8 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -1,6 +1,7 @@ import copy import os import time +from threading import Thread from conan.api.output import ConanOutput @@ -13,6 +14,7 @@ from conans.model.package_ref import PkgReference from conans.paths import EXPORT_SOURCES_TGZ_NAME from conans.util.dates import from_iso8601_to_timestamp +from conans.util.thread import ExceptionThread class RestV2Methods(RestCommonMethods): @@ -43,7 +45,7 @@ def get_recipe(self, ref, dest_folder): # If we didn't indicated reference, server got the latest, use absolute now, it's safer urls = {fn: self.router.recipe_file(ref, fn) for fn in files} - self._download_and_save_files(urls, dest_folder, files) + self._download_and_save_files(urls, dest_folder, files, parallel=True) ret = {fn: os.path.join(dest_folder, fn) for fn in files} return ret @@ -133,7 +135,7 @@ def _upload_files(self, files, urls): raise ConanException("Execute upload again to retry upload the failed files: %s" % ", ".join(failed)) - def _download_and_save_files(self, urls, dest_folder, files): + def _download_and_save_files(self, urls, dest_folder, files, parallel=False): # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first retry = self._config.get("core.download:retry", check_type=int, default=2) @@ -142,12 +144,24 @@ def _download_and_save_files(self, urls, dest_folder, files): if download_cache and not os.path.isabs(download_cache): raise ConanException("core.download:download_cache must be an absolute path") downloader = CachingFileDownloader(self.requester, download_cache=download_cache) + threads = [] + for filename in sorted(files, reverse=True): resource_url = urls[filename] abs_path = os.path.join(dest_folder, filename) os.makedirs(os.path.dirname(abs_path), exist_ok=True) # filename in subfolder must exist - downloader.download(url=resource_url, file_path=abs_path, auth=self.auth, - verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait) + if parallel: + kwargs = {"url": resource_url, "file_path": abs_path, "retry": retry, + "retry_wait": retry_wait, "verify_ssl": self.verify_ssl, + "auth": self.auth} + thread = ExceptionThread(target=downloader.download, kwargs=kwargs) + threads.append(thread) + thread.start() + else: + downloader.download(url=resource_url, file_path=abs_path, auth=self.auth, + verify_ssl=self.verify_ssl, retry=retry, retry_wait=retry_wait) + for t in threads: + t.join() def remove_all_packages(self, ref): """ Remove all packages from the specified reference""" diff --git a/conans/test/integration/cache/cache2_update_test.py b/conans/test/integration/cache/cache2_update_test.py index 35633aee5a9..2effa000fdf 100644 --- a/conans/test/integration/cache/cache2_update_test.py +++ b/conans/test/integration/cache/cache2_update_test.py @@ -256,7 +256,6 @@ def test_revision_fixed_version(self): assert "liba/1.0.0: Checking remote: server0" in self.client.out assert "liba/1.0.0: Checking remote: server1" not in self.client.out assert "liba/1.0.0: Checking remote: server2" not in self.client.out - assert "liba/1.0.0: Trying with 'server0'..." in self.client.out server_rrev_norev = copy.copy(server_rrev) server_rrev_norev.revision = None latest_cache_revision = self.client.cache.get_latest_recipe_reference(server_rrev_norev) diff --git a/conans/test/unittests/tools/microsoft/conanvcvars.bat b/conans/test/unittests/tools/microsoft/conanvcvars.bat deleted file mode 100644 index 416c44db1ae..00000000000 --- a/conans/test/unittests/tools/microsoft/conanvcvars.bat +++ /dev/null @@ -1,2 +0,0 @@ -@echo off -set "VSCMD_START_DIR=%CD%" && call "." amd64 -vcvars_ver=14.3 diff --git a/conans/util/thread.py b/conans/util/thread.py new file mode 100644 index 00000000000..48cfdc9e624 --- /dev/null +++ b/conans/util/thread.py @@ -0,0 +1,15 @@ +from threading import Thread + + +class ExceptionThread(Thread): + def run(self): + self._exc = None + try: + super().run() + except Exception as e: + self._exc = e + + def join(self, timeout=None): + super().join(timeout=timeout) + if self._exc: + raise self._exc diff --git a/conans/util/tracer.py b/conans/util/tracer.py index 4ead57b22b6..917a95b2307 100644 --- a/conans/util/tracer.py +++ b/conans/util/tracer.py @@ -44,18 +44,6 @@ def log_package_upload(pref, duration, files_uploaded, remote): ConanOutput().trace(data) -def log_recipe_download(ref, duration, remote_name, files_downloaded): - assert(isinstance(ref, RecipeReference)) - files_downloaded = files_downloaded or {} - files_downloaded = [_file_document(name, path) for name, path in files_downloaded.items()] - - data = {"_action": "DOWNLOADED_RECIPE", "_id": repr(ref), - "duration": duration, - "remote": remote_name, - "files": files_downloaded} - ConanOutput().trace(data) - - def log_recipe_sources_download(ref, duration, remote_name, files_downloaded): assert(isinstance(ref, RecipeReference)) files_downloaded = files_downloaded or {} @@ -75,12 +63,6 @@ def log_package_download(pref, duration, remote, files_downloaded): ConanOutput().trace(data) -def log_recipe_got_from_local_cache(ref): - assert(isinstance(ref, RecipeReference)) - data = {"_action": "GOT_RECIPE_FROM_LOCAL_CACHE", "_id": repr(ref)} - ConanOutput().trace(data) - - def log_package_got_from_local_cache(pref): assert(isinstance(pref, PkgReference)) tmp = copy.copy(pref) From 9b340972b8d5850d551822c3fa071c2603b63c8d Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Jan 2023 11:27:58 +0100 Subject: [PATCH 26/62] Feature/develop2 info package filters (#12836) * wip * wip * review * fix tests --- conan/cli/commands/graph.py | 4 +- conan/cli/formatters/graph/graph.py | 18 ++++++++- conan/cli/formatters/graph/graph_info_text.py | 40 +++++++++++-------- conans/model/conan_file.py | 1 + .../integration/command/info/info_test.py | 24 ++++++++--- .../command/info/test_graph_info_graphical.py | 4 +- 6 files changed, 64 insertions(+), 27 deletions(-) diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index 5da4c62a36f..ceae5086db8 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -125,8 +125,8 @@ def graph_info(conan_api, parser, subparser, *args): "--requires") if not args.path and not args.requires and not args.tool_requires: raise ConanException("Please specify at least a path to a conanfile or a valid reference.") - if args.format is not None and (args.filter or args.package_filter): - raise ConanException("Formatted outputs cannot be filtered") + if args.format in ("html", "dot") and args.filter: + raise ConanException(f"Formatted output '{args.format}' cannot filter fields") cwd = os.getcwd() path = conan_api.local.get_conanfile_path(args.path, cwd, py=None) if args.path else None diff --git a/conan/cli/formatters/graph/graph.py b/conan/cli/formatters/graph/graph.py index 9a0d75c5f9b..3c6ef38df8e 100644 --- a/conan/cli/formatters/graph/graph.py +++ b/conan/cli/formatters/graph/graph.py @@ -5,6 +5,7 @@ from conan.api.output import cli_out_write +from conan.cli.formatters.graph.graph_info_text import filter_graph from conan.cli.formatters.graph.info_graph_dot import graph_info_dot from conan.cli.formatters.graph.info_graph_html import graph_info_html from conans.client.graph.graph import BINARY_CACHE, \ @@ -99,6 +100,11 @@ def _render_graph(graph, template, template_folder): def format_graph_html(result): graph = result["graph"] conan_api = result["conan_api"] + package_filter = result["package_filter"] + serial = graph.serialize() + # TODO: This is not used, it is necessary to update the renderings to use the serialized graph + # instead of the native graph + serial = filter_graph(serial, package_filter) template_folder = os.path.join(conan_api.cache_folder, "templates") user_template = os.path.join(template_folder, "graph.html") template = load(user_template) if os.path.isfile(user_template) else graph_info_html @@ -110,6 +116,11 @@ def format_graph_html(result): def format_graph_dot(result): graph = result["graph"] conan_api = result["conan_api"] + package_filter = result["package_filter"] + serial = graph.serialize() + # TODO: This is not used, it is necessary to update the renderings to use the serialized graph + # instead of the native graph + serial = filter_graph(serial, package_filter) template_folder = os.path.join(conan_api.cache_folder, "templates") user_template = os.path.join(template_folder, "graph.dot") template = load(user_template) if os.path.isfile(user_template) else graph_info_dot @@ -120,8 +131,11 @@ def format_graph_dot(result): def format_graph_json(result): graph = result["graph"] - serialized = graph.serialize() - json_result = json.dumps(serialized, indent=4) + field_filter = result["field_filter"] + package_filter = result["package_filter"] + serial = graph.serialize() + serial = filter_graph(serial, package_filter, field_filter) + json_result = json.dumps(serial, indent=4) cli_out_write(json_result) if graph.error: raise graph.error diff --git a/conan/cli/formatters/graph/graph_info_text.py b/conan/cli/formatters/graph/graph_info_text.py index 2c99a9d66eb..6c5edc90227 100644 --- a/conan/cli/formatters/graph/graph_info_text.py +++ b/conan/cli/formatters/graph/graph_info_text.py @@ -1,41 +1,49 @@ import fnmatch +from collections import OrderedDict from conan.api.output import ConanOutput +def filter_graph(graph, package_filter, field_filter=None): + if package_filter is not None: + graph["nodes"] = [n for n in graph["nodes"] + if any(fnmatch.fnmatch(n["ref"] or "", p) for p in package_filter)] + if field_filter is not None: + if "ref" not in field_filter: + field_filter.append("ref") + result = [] + for n in graph["nodes"]: + new_node = OrderedDict((k, v) for k, v in n.items() if k in field_filter) + result.append(new_node) + graph["nodes"] = result + return graph + + def format_graph_info(result): - graph = result["graph"] - field_filter = result["field_filter"] - package_filter = result["package_filter"] """ More complete graph output, including information for every node in the graph Used for 'graph info' command """ + graph = result["graph"] + field_filter = result["field_filter"] + package_filter = result["package_filter"] + out = ConanOutput() out.title("Basic graph information") serial = graph.serialize() + serial = filter_graph(serial, package_filter, field_filter) for n in serial["nodes"]: - if package_filter is not None: - display = False - for p in package_filter: - if fnmatch.fnmatch(n["ref"] or "", p): - display = True - break - if not display: - continue out.writeln(f"{n['ref']}:") # FIXME: This can be empty for consumers and it is ugly ":" - _serial_pretty_printer(n, field_filter, indent=" ") + _serial_pretty_printer(n, indent=" ") if graph.error: raise graph.error -def _serial_pretty_printer(data, field_filter, indent=""): +def _serial_pretty_printer(data, indent=""): out = ConanOutput() for k, v in data.items(): - if field_filter is not None and k not in field_filter: - continue if isinstance(v, dict): out.writeln(f"{indent}{k}:") # TODO: increment color too - _serial_pretty_printer(v, None, indent=indent+" ") + _serial_pretty_printer(v, indent=indent+" ") else: out.writeln(f"{indent}{k}: {v}") diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 6eea1be6309..96e4544efee 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -279,6 +279,7 @@ def serialize(self): result["build_folder"] = self.build_folder result["package_folder"] = self.package_folder result["cpp_info"] = self.cpp_info.serialize() + result["label"] = self.display_name return result @property diff --git a/conans/test/integration/command/info/info_test.py b/conans/test/integration/command/info/info_test.py index d9572476a10..fb5758c8b82 100644 --- a/conans/test/integration/command/info/info_test.py +++ b/conans/test/integration/command/info/info_test.py @@ -101,17 +101,31 @@ def test_filter_fields(self): assert "license" in c.out assert "author: myself" in c.out + def test_filter_fields_json(self): + # The --filter arg should work, specifying which fields to show only + c = TestClient() + c.save({"conanfile.py": GenConanfile() + .with_class_attribute("author = 'myself'") + .with_class_attribute("license = 'MIT'") + .with_class_attribute("url = 'http://url.com'")}) + c.run("graph info . --filter=license --format=json") + assert "author" not in c.out + assert '"license": "MIT"' in c.out + c.run("graph info . --filter=license --format=html", assert_error=True) + assert "Formatted output 'html' cannot filter fields" in c.out + class TestJsonOutput: - def test_json_not_filtered(self): + def test_json_package_filter(self): # Formatted output like json or html doesn't make sense to be filtered client = TestClient() conanfile = GenConanfile("pkg", "0.1").with_setting("build_type") client.save({"conanfile.py": conanfile}) - client.run("graph info . --filter=license --format=json", assert_error=True) - assert "Formatted outputs cannot be filtered" in client.out - client.run("graph info . --package-filter=license --format=html", assert_error=True) - assert "Formatted outputs cannot be filtered" in client.out + client.run("graph info . --package-filter=nothing --format=json") + assert '"nodes": []' in client.out + client.run("graph info . --package-filter=pkg* --format=json") + graph = json.loads(client.stdout) + assert graph["nodes"][0]["ref"] == "pkg/0.1" def test_json_info_outputs(self): client = TestClient() diff --git a/conans/test/integration/command/info/test_graph_info_graphical.py b/conans/test/integration/command/info/test_graph_info_graphical.py index f70b85cd6d7..0d4a52e9aec 100644 --- a/conans/test/integration/command/info/test_graph_info_graphical.py +++ b/conans/test/integration/command/info/test_graph_info_graphical.py @@ -173,7 +173,7 @@ def test_user_templates(): template_folder = os.path.join(c.cache_folder, 'templates') c.save({"graph.html": '{{ base_template_path }}', "graph.dot": '{{ base_template_path }}'}, path=template_folder) - c.run("graph info --requires=app/0.1 --format=html", assert_error=True) + c.run("graph info --requires=lib/0.1 --format=html") assert template_folder in c.stdout - c.run("graph info --requires=app/0.1 --format=dot", assert_error=True) + c.run("graph info --requires=lib/0.1 --format=dot") assert template_folder in c.stdout From ec60f1f189322a52e1779d1c5c843516c9fd17ff Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Jan 2023 11:55:06 +0100 Subject: [PATCH 27/62] Fixing cmake_layout single config (#12829) * fixing cmake_layout single config * fix tests * fix tests * fix tests --- conan/tools/cmake/layout.py | 32 +++++++--------- conan/tools/cmake/presets.py | 10 ++++- .../test_build_system_layout_helpers.py | 5 ++- .../functional/layout/test_local_commands.py | 6 ++- .../cmake/cmakedeps/test_cmakedeps.py | 4 +- .../test_cmakedeps_and_linker_flags.py | 3 +- .../toolchains/cmake/test_cmake_toolchain.py | 23 ++++++----- .../cmake/test_cmake_toolchain_xcode_flags.py | 6 +-- .../cmake/cmakedeps/test_cmakedeps.py | 33 +++++++++------- .../toolchains/cmake/test_cmaketoolchain.py | 38 ++++++++++++++++++- 10 files changed, 107 insertions(+), 53 deletions(-) diff --git a/conan/tools/cmake/layout.py b/conan/tools/cmake/layout.py index f39ef697472..abddc047f38 100644 --- a/conan/tools/cmake/layout.py +++ b/conan/tools/cmake/layout.py @@ -22,20 +22,21 @@ def cmake_layout(conanfile, generator=None, src_folder=".", build_folder="build" raise ConanException("'build_type' setting not defined, it is necessary for cmake_layout()") build_folder = build_folder if not subproject else os.path.join(subproject, build_folder) - custom_conf = get_build_folder_custom_vars(conanfile) - if custom_conf: - build_folder = os.path.join(build_folder, custom_conf) + config_build_folder, user_defined_build = get_build_folder_custom_vars(conanfile) - if multi: - conanfile.folders.build = build_folder + if config_build_folder: + conanfile.folders.build = os.path.join(build_folder, config_build_folder) else: - conanfile.folders.build = os.path.join(build_folder, build_type) + if not multi and not user_defined_build: + conanfile.folders.build = os.path.join(build_folder, build_type) + else: + conanfile.folders.build = build_folder - conanfile.folders.generators = os.path.join(build_folder, "generators") + conanfile.folders.generators = os.path.join(conanfile.folders.build, "generators") conanfile.cpp.source.includedirs = ["include"] - if multi: + if multi and not user_defined_build: conanfile.cpp.build.libdirs = ["{}".format(build_type)] conanfile.cpp.build.bindirs = ["{}".format(build_type)] else: @@ -49,17 +50,11 @@ def get_build_folder_custom_vars(conanfile): default=[], check_type=list) ret = [] for s in build_vars: + group, var = s.split(".", 1) tmp = None - if s.startswith("settings."): - _, var = s.split("settings.", 1) - if var == "build_type": - raise ConanException("Error, don't include 'settings.build_type' in the " - "'tools.cmake.cmake_layout:build_folder_vars' conf. It is " - "managed by default because 'CMakeToolchain' and 'CMakeDeps' " - "are multi-config generators.`") + if group == "settings": tmp = conanfile.settings.get_safe(var) - elif s.startswith("options."): - _, var = s.split("options.", 1) + elif group == "options": value = conanfile.options.get_safe(var) if value is not None: tmp = "{}_{}".format(var, value) @@ -69,4 +64,5 @@ def get_build_folder_custom_vars(conanfile): if tmp: ret.append(tmp.lower()) - return "-".join(ret) + user_defined_build = "settings.build_type" in build_vars + return "-".join(ret), user_defined_build diff --git a/conan/tools/cmake/presets.py b/conan/tools/cmake/presets.py index 6d4a9a73587..dacf021effd 100644 --- a/conan/tools/cmake/presets.py +++ b/conan/tools/cmake/presets.py @@ -21,7 +21,10 @@ def _build_preset(conanfile, multiconfig): def _build_preset_name(conanfile): build_type = conanfile.settings.get_safe("build_type") - custom_conf = get_build_folder_custom_vars(conanfile) + custom_conf, user_defined_build = get_build_folder_custom_vars(conanfile) + if user_defined_build: + return custom_conf + if custom_conf: if build_type: return "{}-{}".format(custom_conf, build_type.lower()) @@ -32,7 +35,10 @@ def _build_preset_name(conanfile): def _configure_preset_name(conanfile, multiconfig): build_type = conanfile.settings.get_safe("build_type") - custom_conf = get_build_folder_custom_vars(conanfile) + custom_conf, user_defined_build = get_build_folder_custom_vars(conanfile) + + if user_defined_build: + return custom_conf if multiconfig or not build_type: return "default" if not custom_conf else custom_conf diff --git a/conans/test/functional/layout/test_build_system_layout_helpers.py b/conans/test/functional/layout/test_build_system_layout_helpers.py index 4bd7c8cd4d0..c683114b8c3 100644 --- a/conans/test/functional/layout/test_build_system_layout_helpers.py +++ b/conans/test/functional/layout/test_build_system_layout_helpers.py @@ -168,7 +168,8 @@ def package(self): # Local flow client.run("install . foo/1.0 -s os=Linux") - assert os.path.exists(os.path.join(client.current_folder, "build", "generators", "generate.txt")) + assert os.path.exists(os.path.join(client.current_folder, + "build", "Release", "generators", "generate.txt")) client.run("source .") assert os.path.exists(os.path.join(client.current_folder, "src", "source.txt")) client.run("build .") @@ -294,4 +295,4 @@ def layout(self): client.save({"conanfile.py": conanfile}) client.run("install .") assert os.path.exists(os.path.join(client.current_folder, - "mybuild/generators/conan_toolchain.cmake")) + "mybuild/Release/generators/conan_toolchain.cmake")) diff --git a/conans/test/functional/layout/test_local_commands.py b/conans/test/functional/layout/test_local_commands.py index bd036d617db..0c07d5cdd1b 100644 --- a/conans/test/functional/layout/test_local_commands.py +++ b/conans/test/functional/layout/test_local_commands.py @@ -372,7 +372,11 @@ def test_start_dir_failure(): c = TestClient() c.save(pkg_cmake("dep", "0.1")) c.run("install .") - expected_path = os.path.join(c.current_folder, "build", "generators", "conan_toolchain.cmake") + if platform.system() != "Windows": + gen_folder = os.path.join(c.current_folder, "build", "Release", "generators") + else: + gen_folder = os.path.join(c.current_folder, "build", "generators") + expected_path = os.path.join(gen_folder, "conan_toolchain.cmake") assert os.path.exists(expected_path) os.unlink(expected_path) with c.chdir("build"): diff --git a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps.py b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps.py index 45b5d1d97e6..c1ceb5d3a0e 100644 --- a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps.py +++ b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps.py @@ -497,7 +497,7 @@ def package_info(self): files = pkg_cmake("mylib", "0.1", requires=["zlib/0.1"]) files["CMakeLists.txt"] = files["CMakeLists.txt"].replace("find_package(zlib)", "find_package(ZLIB)") - files["CMakeLists.txt"] = files["CMakeLists.txt"].replace("zlib::zlib","ZLIB::ZLIB") + files["CMakeLists.txt"] = files["CMakeLists.txt"].replace("zlib::zlib", "ZLIB::ZLIB") client.save({os.path.join("mylib", name): content for name, content in files.items()}) files = pkg_cmake("consumer", "0.1", requires=["mylib/0.1"]) client.save({os.path.join("consumer", name): content for name, content in files.items()}) @@ -509,7 +509,7 @@ def package_info(self): client.run("install consumer") if platform.system() != "Windows": host_arch = client.get_default_host_profile().settings['arch'] - data = os.path.join(f"consumer/build/generators/mylib-release-{host_arch}-data.cmake") + data = f"consumer/build/Release/generators/mylib-release-{host_arch}-data.cmake" contents = client.load(data) assert 'set(ZLIB_FIND_MODE "")' in contents diff --git a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_and_linker_flags.py b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_and_linker_flags.py index e030070541a..06c6d1943ad 100644 --- a/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_and_linker_flags.py +++ b/conans/test/functional/toolchains/cmake/cmakedeps/test_cmakedeps_and_linker_flags.py @@ -52,7 +52,8 @@ def package_info(self): client.run("new hello/1.0 -m cmake_lib") client.save({"conanfile.py": conanfile}) client.run("create .") - t = os.path.join("test_package", "build", "generators", "hello-release-x86_64-data.cmake") + t = os.path.join("test_package", "build", "Release", + "generators", "hello-release-x86_64-data.cmake") target_data_cmake_content = client.load(t) assert 'set(hello_SHARED_LINK_FLAGS_RELEASE "-z now;-z relro")' in target_data_cmake_content assert 'set(hello_EXE_LINK_FLAGS_RELEASE "-z now;-z relro")' in target_data_cmake_content diff --git a/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py b/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py index 49f73cf5a2a..dbd70681ef0 100644 --- a/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py +++ b/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py @@ -297,8 +297,14 @@ def test_install_output_directories(): p_folder = client.cache.package_layout(pref.ref).package(pref) assert os.path.exists(os.path.join(p_folder, "mylibs")) assert not os.path.exists(os.path.join(p_folder, "lib")) + b_folder = client.cache.package_layout(pref.ref).build(pref) - toolchain = client.load(os.path.join(b_folder, "build", "generators", "conan_toolchain.cmake")) + if platform.system() != "Windows": + gen_folder = os.path.join(b_folder, "build", "Release", "generators") + else: + gen_folder = os.path.join(b_folder, "build", "generators") + + toolchain = client.load(os.path.join(gen_folder, "conan_toolchain.cmake")) assert 'set(CMAKE_INSTALL_LIBDIR "mylibs")' in toolchain @@ -962,15 +968,15 @@ def test_cmake_presets_with_conanfile_txt(): assert "Hello World Release!" in c.out -def test_cmake_presets_forbidden_build_type(): +def test_cmake_presets_not_forbidden_build_type(): client = TestClient(path_with_spaces=False) client.run("new hello/0.1 --template cmake_exe") # client.run("new cmake_exe -d name=hello -d version=0.1") settings_layout = '-c tools.cmake.cmake_layout:build_folder_vars=' \ '\'["options.missing", "settings.build_type"]\'' - client.run("install . {}".format(settings_layout), assert_error=True) - assert "Error, don't include 'settings.build_type' in the " \ - "'tools.cmake.cmake_layout:build_folder_vars' conf" in client.out + client.run("install . {}".format(settings_layout)) + assert os.path.exists(os.path.join(client.current_folder, + "build/release/generators/conan_toolchain.cmake")) def test_resdirs_cmake_install(): @@ -1116,7 +1122,8 @@ def test_cmake_toolchain_vars_when_option_declared(): # the CMakeLists fpic_option = "-o mylib:fPIC=False" if platform.system() != "Windows" else "" t.run(f"install . -o mylib:shared=False {fpic_option}") - t.run_command("cmake -S . -B build/ -DCMAKE_TOOLCHAIN_FILE=build/generators/conan_toolchain.cmake") + folder = "build/generators" if platform.system() == "Windows" else "build/Release/generators" + t.run_command(f"cmake -S . -B build/ -DCMAKE_TOOLCHAIN_FILE={folder}/conan_toolchain.cmake") assert "mylib target type: STATIC_LIBRARY" in t.out assert f"mylib position independent code: OFF" in t.out @@ -1185,8 +1192,6 @@ def package(self): "build_profile": build_profile }) - xxx = client.get_default_build_profile() - client.run("create . -pr:b build_profile -pr:h build_profile") client.run("create . -pr:b build_profile -pr:h host_profile") @@ -1226,7 +1231,7 @@ def build_requirements(self): host_context_package_id = "bf544cd3bc20b82121fd76b82eacbb36d75fa167" with client.chdir("build"): - client.run_command("cmake .. -DCMAKE_TOOLCHAIN_FILE=generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release") + client.run_command("cmake .. -DCMAKE_TOOLCHAIN_FILE=Release/generators/conan_toolchain.cmake -DCMAKE_BUILD_TYPE=Release") # Verify binary executable is found from build context package, # and library comes from host context package assert f"package/{build_context_package_id}/bin/foobin" in client.out diff --git a/conans/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py b/conans/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py index 30b5622b2a5..f108945ef56 100644 --- a/conans/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py +++ b/conans/test/functional/toolchains/cmake/test_cmake_toolchain_xcode_flags.py @@ -38,7 +38,7 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_enabled(op_system, os_vers client.run("new hello/0.1 --template=cmake_lib") _add_message_status_flags(client) client.run("install . --profile:build=default --profile:host=host") - toolchain = client.load(os.path.join("build", "generators", "conan_toolchain.cmake")) + toolchain = client.load(os.path.join("build", "Release", "generators", "conan_toolchain.cmake")) # bitcode assert 'set(CMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE "YES")' in toolchain assert 'set(CMAKE_XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE "bitcode")' in toolchain @@ -121,7 +121,7 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_disabled(op_system, os_ver client.run("new hello/0.1 --template=cmake_lib") _add_message_status_flags(client) client.run("install . --profile:build=default --profile:host=host") - toolchain = client.load(os.path.join("build", "generators", "conan_toolchain.cmake")) + toolchain = client.load(os.path.join("build", "Release", "generators", "conan_toolchain.cmake")) # bitcode assert 'set(CMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE "NO")' in toolchain assert 'set(CMAKE_XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE "bitcode")' not in toolchain @@ -163,7 +163,7 @@ def test_cmake_apple_bitcode_arc_and_visibility_flags_are_none(op_system, os_ver client.run("new hello/0.1 --template=cmake_lib") _add_message_status_flags(client) client.run("install . --profile:build=default --profile:host=host") - toolchain = client.load(os.path.join("build", "generators", "conan_toolchain.cmake")) + toolchain = client.load(os.path.join("build", "Release", "generators", "conan_toolchain.cmake")) # bitcode assert 'set(CMAKE_XCODE_ATTRIBUTE_ENABLE_BITCODE "NO")' not in toolchain assert 'set(CMAKE_XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE "bitcode")' not in toolchain diff --git a/conans/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py b/conans/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py index 6d3e9b57d6e..b3da541f9ec 100644 --- a/conans/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py +++ b/conans/test/integration/toolchains/cmake/cmakedeps/test_cmakedeps.py @@ -1,4 +1,5 @@ import os +import platform import textwrap from conans.test.assets.genconanfile import GenConanfile @@ -225,9 +226,14 @@ def generate(self): client.save({"foo.py": foo.format(set_find_mode=""), "bar.py": bar}, clean_first=True) - module_file = os.path.join(client.current_folder, "build", "generators", "module-custom_bar_module_file_nameTargets.cmake") - components_module = os.path.join(client.current_folder, "build", "generators", "custom_bar_file_name-Target-release.cmake") - config_file = os.path.join(client.current_folder, "build", "generators", "custom_bar_file_nameTargets.cmake") + if platform.system() != "Windows": + gen_folder = os.path.join(client.current_folder, "build", "Release", "generators") + else: + gen_folder = os.path.join(client.current_folder, "build", "generators") + + module_file = os.path.join(gen_folder, "module-custom_bar_module_file_nameTargets.cmake") + components_module = os.path.join(gen_folder, "custom_bar_file_name-Target-release.cmake") + config_file = os.path.join(gen_folder, "custom_bar_file_nameTargets.cmake") # uses cmake_find_mode set in bar: both client.run("create bar.py bar/1.0@") @@ -311,19 +317,20 @@ def generate(self): client.save({"foo.py": foo.format(set_find_mode=""), "bar.py": bar}, clean_first=True) - module_file = os.path.join(client.current_folder, "build", "generators", - "module-custom_bar_module_file_nameTargets.cmake") - components_module = os.path.join(client.current_folder, "build", "generators", - "custom_bar_file_name-Target-release.cmake") - config_file = os.path.join(client.current_folder, "build", "generators", - "custom_bar_file_nameTargets.cmake") + if platform.system() != "Windows": + gen_folder = os.path.join(client.current_folder, "build", "Release", "generators") + else: + gen_folder = os.path.join(client.current_folder, "build", "generators") + + module_file = os.path.join(gen_folder, "module-custom_bar_module_file_nameTargets.cmake") + components_module = os.path.join(gen_folder, "custom_bar_file_name-Target-release.cmake") + config_file = os.path.join(gen_folder, "custom_bar_file_nameTargets.cmake") - module_file_build = os.path.join(client.current_folder, "build", "generators", + module_file_build = os.path.join(gen_folder, "module-custom_bar_build_module_file_name_BUILDTargets.cmake") - components_module_build = os.path.join(client.current_folder, "build", "generators", + components_module_build = os.path.join(gen_folder, "custom_bar_build_file_name_BUILD-Target-release.cmake") - config_file_build = os.path.join(client.current_folder, "build", "generators", - "custom_bar_build_file_name_BUILDTargets.cmake") + config_file_build = os.path.join(gen_folder, "custom_bar_build_file_name_BUILDTargets.cmake") # uses cmake_find_mode set in bar: both client.run("create bar.py bar/1.0@ -pr:h=default -pr:b=default") diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index e219bcc5a06..38a793f427a 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -8,7 +8,6 @@ from conan.tools.cmake.presets import load_cmake_presets from conans.test.assets.genconanfile import GenConanfile -from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient from conans.util.files import rmdir @@ -379,7 +378,7 @@ def layout(self): else: build_dir = os.path.join(client.current_folder, "build") - presets = load_cmake_presets(os.path.join(client.current_folder, "build", "generators")) + presets = load_cmake_presets(os.path.join(build_dir, "generators")) assert presets["configurePresets"][0]["binaryDir"] == build_dir @@ -422,6 +421,7 @@ def test_cmake_presets_multiconfig(): # Repeat one client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") + presets = json.loads(client.load("CMakePresets.json")) assert len(presets["buildPresets"]) == 4 assert presets["buildPresets"][0]["configuration"] == "Release" assert presets["buildPresets"][1]["configuration"] == "Debug" @@ -517,6 +517,7 @@ def _format_val(val): assert "-DCMAKE_TOOLCHAIN_FILE=" in client.out assert f"-G {_format_val('MinGW Makefiles')}" in client.out + def test_android_c_library(): client = TestClient() conanfile = textwrap.dedent(""" @@ -845,3 +846,36 @@ def test_set_cmake_lang_compilers_and_launchers(): assert 'set(CMAKE_C_COMPILER "/my/local/gcc")' in toolchain assert 'set(CMAKE_CXX_COMPILER "g++")' in toolchain assert 'set(CMAKE_RC_COMPILER "C:/local/rc.exe")' in toolchain + + +def test_cmake_layout_toolchain_folder(): + """ in single-config generators, the toolchain is a different file per configuration + https://github.com/conan-io/conan/issues/12827 + """ + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import cmake_layout + + class Conan(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeToolchain" + + def layout(self): + cmake_layout(self) + """) + c.save({"conanfile.py": conanfile}) + c.run("install . -s os=Linux -s compiler=gcc -s compiler.version=7 -s build_type=Release " + "-s compiler.libcxx=libstdc++11") + assert os.path.exists(os.path.join(c.current_folder, + "build/Release/generators/conan_toolchain.cmake")) + c.run("install . -s os=Linux -s compiler=gcc -s compiler.version=7 -s build_type=Debug " + "-s compiler.libcxx=libstdc++11") + assert os.path.exists(os.path.join(c.current_folder, + "build/Debug/generators/conan_toolchain.cmake")) + c.run("install . -s os=Linux -s compiler=gcc -s compiler.version=7 -s build_type=Debug " + "-s compiler.libcxx=libstdc++11 -s arch=x86 " + "-c tools.cmake.cmake_layout:build_folder_vars='[\"settings.arch\", \"settings.build_type\"]'") + print(c.out) + assert os.path.exists(os.path.join(c.current_folder, + "build/x86-debug/generators/conan_toolchain.cmake")) From c539f450e246b7e05ce16a59068f8e296e4487ec Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Jan 2023 12:11:10 +0100 Subject: [PATCH 28/62] test to check overrides work fine (#8084) * test to check overrides work fine * updated test --- .../version_range_override_test.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/conans/test/integration/graph/version_ranges/version_range_override_test.py b/conans/test/integration/graph/version_ranges/version_range_override_test.py index c861a399298..0a23c78b3ef 100644 --- a/conans/test/integration/graph/version_ranges/version_range_override_test.py +++ b/conans/test/integration/graph/version_ranges/version_range_override_test.py @@ -90,3 +90,45 @@ def test(self, host_context): lock = t.load("conan.lock") self.assertIn("gtest/1.8.1@bloomberg/stable", lock) self.assertNotIn("gtest/1.8.0@PORT/stable", lock) + + def test_override(self): + """ + pkga -> ros_perception -> ros_core + \\-----> pkgb -----------/ + """ + # https://github.com/conan-io/conan/issues/8071 + t = TestClient() + t.save({"conanfile.py": GenConanfile()}) + t.run("create . ros_core/1.1.4@3rdparty/unstable") + t.run("create . ros_core/PR-53@3rdparty/snapshot") + t.save({"conanfile.py": GenConanfile().with_requires("ros_core/1.1.4@3rdparty/unstable")}) + t.run("create . ros_perception/1.1.4@3rdparty/unstable") + t.run("create . ros_perception/PR-53@3rdparty/snapshot") + t.save({"conanfile.py": GenConanfile().with_requires("ros_core/[~=1.1]@3rdparty/unstable")}) + t.run("create . pkgb/0.1@common/unstable") + t.save({"conanfile.py": GenConanfile("pkga", "0.1").with_requires( + "ros_perception/[~=1.1]@3rdparty/unstable", + "pkgb/[~=0]@common/unstable")}) + t.run("create . ") + self.assertIn("ros_core/1.1.4@3rdparty/unstable", t.out) + self.assertIn("ros_perception/1.1.4@3rdparty/unstable", t.out) + self.assertNotIn("snapshot", t.out) + + t.save({"conanfile.py": GenConanfile("pkga", "0.1") + .with_require("pkgb/[~=0]@common/unstable") + .with_require("ros_perception/PR-53@3rdparty/snapshot") + .with_require("ros_core/PR-53@3rdparty/snapshot", override=True)}) + + t.run("create . --build=missing --build=pkga") + self.assertIn("ros_core/PR-53@3rdparty/snapshot", t.out) + self.assertIn("ros_perception/PR-53@3rdparty/snapshot", t.out) + + # Override only the upstream without overriding the direct one + t.save({"conanfile.py": GenConanfile("pkga", "0.1") + .with_require("pkgb/[~=0]@common/unstable") + .with_require("ros_perception/[~=1.1]@3rdparty/unstable") + .with_require("ros_core/PR-53@3rdparty/snapshot", override=True)}) + + t.run("create . --build=missing --build=pkga") + self.assertIn("ros_core/PR-53@3rdparty/snapshot", t.out) + self.assertIn("ros_perception/1.1.4@3rdparty/unstable", t.out) From 904162523b2947998919bb6a84ece6ec26e5730e Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Jan 2023 12:14:01 +0100 Subject: [PATCH 29/62] [develop2] conditional requirement on context (#12851) --- .../build_requires/build_requires_test.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/conans/test/integration/build_requires/build_requires_test.py b/conans/test/integration/build_requires/build_requires_test.py index 26454e6732f..e64c4690791 100644 --- a/conans/test/integration/build_requires/build_requires_test.py +++ b/conans/test/integration/build_requires/build_requires_test.py @@ -726,3 +726,25 @@ def config_options(self): c.assert_listed_binary({"pkg/0.1": ("62e589af96a19807968167026d906e63ed4de1f5", "Build")}, build=True) assert "Finalizing install" in c.out + + +def test_conditional_require_context(): + """ test that we can condition on the context to define a dependency + """ + c = TestClient() + pkg = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + name = "pkg" + version = "0.1" + def requirements(self): + if self.context == "host": + self.requires("dep/1.0") + """) + c.save({"dep/conanfile.py": GenConanfile("dep", "1.0"), + "consumer/conanfile.py": pkg}) + c.run("create dep") + c.run("create consumer") + c.assert_listed_require({"dep/1.0": "Cache"}) + c.run("create consumer --build-require") + assert "dep/1.0" not in c.out From 6e1f2bb60a430f2567480ddeaba19cafb20fbcb2 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 5 Jan 2023 15:50:39 +0100 Subject: [PATCH 30/62] fix XcodeDeps in editable root path (#12841) * fix XcodeDeps in editable root path * fix test --- conan/tools/apple/xcodedeps.py | 9 +++++++-- conans/test/integration/layout/test_layout_paths.py | 13 +++++++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/conan/tools/apple/xcodedeps.py b/conan/tools/apple/xcodedeps.py index fb79a0259d7..f4b6d92094d 100644 --- a/conan/tools/apple/xcodedeps.py +++ b/conan/tools/apple/xcodedeps.py @@ -130,6 +130,7 @@ def _merged_vars(name): merged = [var for cpp_info in transitive_cpp_infos for var in getattr(cpp_info, name)] return list(OrderedDict.fromkeys(merged).keys()) + # TODO: Investigate if paths can be made relative to "root" folder fields = { 'pkg_name': pkg_name, 'comp_name': comp_name, @@ -270,8 +271,10 @@ def _transitive_components(component): transitive_internal = list(OrderedDict.fromkeys(transitive_internal).keys()) transitive_external = list(OrderedDict.fromkeys(transitive_external).keys()) + # In case dep is editable and package_folder=None + pkg_folder = dep.package_folder or dep.recipe_folder component_content = self.get_content_for_component(dep_name, comp_name, - dep.package_folder, + pkg_folder, transitive_internal, transitive_external) include_components_names.append((dep_name, comp_name)) @@ -289,7 +292,9 @@ def _transitive_components(component): public_deps.append((_format_name(d.ref.name),) * 2) required_components = dep.cpp_info.required_components if dep.cpp_info.required_components else public_deps - root_content = self.get_content_for_component(dep_name, dep_name, dep.package_folder, [dep.cpp_info], + # In case dep is editable and package_folder=None + pkg_folder = dep.package_folder or dep.recipe_folder + root_content = self.get_content_for_component(dep_name, dep_name, pkg_folder, [dep.cpp_info], required_components) include_components_names.append((dep_name, dep_name)) result.update(root_content) diff --git a/conans/test/integration/layout/test_layout_paths.py b/conans/test/integration/layout/test_layout_paths.py index 5d863f411f7..5049ac53440 100644 --- a/conans/test/integration/layout/test_layout_paths.py +++ b/conans/test/integration/layout/test_layout_paths.py @@ -1,3 +1,4 @@ +import os import textwrap from conans.test.assets.genconanfile import GenConanfile @@ -6,6 +7,7 @@ def test_editable_layout_paths(): # https://github.com/conan-io/conan/issues/12521 + # https://github.com/conan-io/conan/issues/12839 c = TestClient() dep = textwrap.dedent(""" import os @@ -21,13 +23,16 @@ def layout(self): "pkg/conanfile.py": GenConanfile("pkg", "0.1").with_settings("build_type", "arch") .with_requires("dep/0.1") .with_generator("CMakeDeps") - .with_generator("PkgConfigDeps")}) + .with_generator("PkgConfigDeps") + .with_generator("XcodeDeps")}) c.run("editable add dep dep/0.1") - c.run("install pkg") + c.run("install pkg -s arch=x86_64") # It doesn't crash anymore assert "dep/0.1:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 - Editable" in c.out - arch = c.get_default_host_profile().settings['arch'] - data = c.load(f"dep-release-{arch}-data.cmake") + data = c.load(f"dep-release-x86_64-data.cmake") assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/include")' in data pc = c.load("dep.pc") assert "includedir1=${prefix}/include" in pc + xcode = c.load("conan_dep_dep_release_x86_64.xcconfig") + dep_path = os.path.join(c.current_folder, "dep") + assert f"PACKAGE_ROOT_dep[config=Release][arch=x86_64][sdk=*] = {dep_path}" in xcode From c0bca6e65aa04adfef76cb5b477689b289f098ec Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Jan 2023 02:21:54 +0100 Subject: [PATCH 31/62] fix broken tests after develop merge --- conans/model/requires.py | 2 ++ conans/test/integration/toolchains/cmake/test_cmaketoolchain.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/conans/model/requires.py b/conans/model/requires.py index e7c23a0be2c..533d0e18d76 100644 --- a/conans/model/requires.py +++ b/conans/model/requires.py @@ -334,6 +334,8 @@ def transform_downstream(self, pkg_type, require, dep_pkg_type): if self.test: downstream_require.test = True + # If the current one is resolving conflicts, the downstream one will be too + downstream_require.force = require.force downstream_require.direct = False return downstream_require diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index b0519625baf..5da7fec1991 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -416,7 +416,6 @@ def test_cmake_presets_multiconfig(): assert presets["buildPresets"][2]["configuration"] == "RelWithDebInfo" assert presets["buildPresets"][3]["configuration"] == "MinSizeRel" - # Repeat one client.run("install --requires=mylib/1.0@ -g CMakeToolchain " "-s build_type=Debug --profile:h=profile") From 72d323ee74be5460ad97f3329e99f68192a9120a Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Jan 2023 02:34:48 +0100 Subject: [PATCH 32/62] fix OSX test --- .../test/integration/remote/rest_api_test.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index fbe8abff11c..8697cfcc008 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -100,18 +100,16 @@ def test_get_package(self): self.api.get_package(pref, tmp_dir) self.assertIn("hello.cpp", os.listdir(tmp_dir)) + @pytest.mark.skipif(platform.system() != "Linux", reason="only Linux") def test_upload_huge_conan(self): - if platform.system() != "Windows": - # Upload a conans - ref = RecipeReference.loads("conanhuge/1.0.0@private_user/testing#myreciperev") - files = {"file%s.cpp" % name: "File conent" for name in range(1000)} - self._upload_recipe(ref, files) - - # Get the conans - tmp = temp_folder() - files = self.api.get_recipe(ref, tmp) - self.assertIsNotNone(files) - self.assertTrue(os.path.exists(os.path.join(tmp, "file999.cpp"))) + ref = RecipeReference.loads("conanhuge/1.0.0@private_user/testing#myreciperev") + files = {"file%s.cpp" % name: "File conent" for name in range(1000)} + self._upload_recipe(ref, files) + + tmp = temp_folder() + files = self.api.get_recipe(ref, tmp) + self.assertIsNotNone(files) + self.assertTrue(os.path.exists(os.path.join(tmp, "file999.cpp"))) def test_search(self): # Upload a conan1 From e91f43cc362c0efcad62cb2274b79f9e79b4921e Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Jan 2023 02:48:40 +0100 Subject: [PATCH 33/62] fixing test --- conans/test/integration/layout/test_layout_paths.py | 2 ++ conans/test/integration/remote/rest_api_test.py | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conans/test/integration/layout/test_layout_paths.py b/conans/test/integration/layout/test_layout_paths.py index 77b252280c2..2c627d026e4 100644 --- a/conans/test/integration/layout/test_layout_paths.py +++ b/conans/test/integration/layout/test_layout_paths.py @@ -31,6 +31,8 @@ def layout(self): assert "dep/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709 - Editable" in c.out arch = c.get_default_host_profile().settings['arch'] + print(os.listdir(c.current_folder)) + print(os.listdir(os.path.join(c.current_folder, "pkg"))) data = c.load(f"pkg/dep-release-{arch}-data.cmake") assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/include")' in data diff --git a/conans/test/integration/remote/rest_api_test.py b/conans/test/integration/remote/rest_api_test.py index 8697cfcc008..11bb13b3a98 100644 --- a/conans/test/integration/remote/rest_api_test.py +++ b/conans/test/integration/remote/rest_api_test.py @@ -13,7 +13,6 @@ from conans.model.conf import ConfDefinition from conans.util.env import environment_update from conans.client.userio import UserInput -from conans.model.info import ConanInfo, load_binary_info from conans.model.manifest import FileTreeManifest from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference @@ -103,13 +102,13 @@ def test_get_package(self): @pytest.mark.skipif(platform.system() != "Linux", reason="only Linux") def test_upload_huge_conan(self): ref = RecipeReference.loads("conanhuge/1.0.0@private_user/testing#myreciperev") - files = {"file%s.cpp" % name: "File conent" for name in range(1000)} + files = {"file%s.cpp" % name: "File conent" for name in range(10)} self._upload_recipe(ref, files) tmp = temp_folder() files = self.api.get_recipe(ref, tmp) self.assertIsNotNone(files) - self.assertTrue(os.path.exists(os.path.join(tmp, "file999.cpp"))) + self.assertTrue(os.path.exists(os.path.join(tmp, "file9.cpp"))) def test_search(self): # Upload a conan1 From 2cb1c49c8d524c412e1b350a194ec342c57fce46 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Jan 2023 03:01:11 +0100 Subject: [PATCH 34/62] ffix test --- conans/test/integration/layout/test_layout_paths.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/conans/test/integration/layout/test_layout_paths.py b/conans/test/integration/layout/test_layout_paths.py index 2c627d026e4..1a4efb741d3 100644 --- a/conans/test/integration/layout/test_layout_paths.py +++ b/conans/test/integration/layout/test_layout_paths.py @@ -30,10 +30,7 @@ def layout(self): # It doesn't crash anymore assert "dep/0.1:da39a3ee5e6b4b0d3255bfef95601890afd80709 - Editable" in c.out - arch = c.get_default_host_profile().settings['arch'] - print(os.listdir(c.current_folder)) - print(os.listdir(os.path.join(c.current_folder, "pkg"))) - data = c.load(f"pkg/dep-release-{arch}-data.cmake") + data = c.load(f"pkg/dep-release-x86_64-data.cmake") assert 'set(dep_INCLUDE_DIRS_RELEASE "${dep_PACKAGE_FOLDER_RELEASE}/include")' in data pc = c.load("pkg/dep.pc") From 456fb44e129f49a3c439070692c0b3f12d57e1d0 Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 9 Jan 2023 09:27:49 +0100 Subject: [PATCH 35/62] fix broken test after develop merge --- conans/test/functional/layout/test_local_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conans/test/functional/layout/test_local_commands.py b/conans/test/functional/layout/test_local_commands.py index cb7d92f4103..a2b23ff8fbe 100644 --- a/conans/test/functional/layout/test_local_commands.py +++ b/conans/test/functional/layout/test_local_commands.py @@ -1,4 +1,5 @@ import os +import platform import re import textwrap From 3da92bdd30dfd32c7369d3013d54978a19bb85b9 Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 9 Jan 2023 10:28:10 +0100 Subject: [PATCH 36/62] Add verbose output for upload to test.pypi (#12867) use verbose output --- .ci/jenkins/testsv2.jenkins | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/jenkins/testsv2.jenkins b/.ci/jenkins/testsv2.jenkins index 07cd109b875..79f4b30f9f3 100644 --- a/.ci/jenkins/testsv2.jenkins +++ b/.ci/jenkins/testsv2.jenkins @@ -179,7 +179,7 @@ void deployToPypiTest() { def deployConan = "python -m pip install twine && " \ + "python .ci/bump_dev_version.py && " \ + "rm -rf dist/ && python setup.py sdist && " \ - + "python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*" + + "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" sh(script: deployConan) } } @@ -187,7 +187,7 @@ void deployToPypiTest() { withCredentials([string(credentialsId: 'TWINE_USERNAME_SERVER', variable: 'TWINE_USERNAME'), string(credentialsId: 'TWINE_PASSWORD_SERVER', variable: 'TWINE_PASSWORD')]) { def deployServer = "rm -rf dist/ rm setup.py && mv setup_server.py setup.py && python setup.py sdist && " \ - + "python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*" + + "python -m twine upload --verbose --repository-url https://test.pypi.org/legacy/ dist/*" sh(script: deployServer) } } From 0a594cf144ad6de3bd6c3ebe87d5a0766a5885aa Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 9 Jan 2023 11:51:39 +0100 Subject: [PATCH 37/62] Fix deploy to test.pypi caused by malformed table in README (#12869) fix malformed table --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index ca6229e0afa..7ab530b4820 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ This is the **developer/maintainer** documentation. For user documentation, go t +-------------------------+ -| **develop2** | +| **develop2** | +=========================+ | |Build Status Develop| | +-------------------------+ From d82e03377d92a11ae67a7bc652eb2b1b237a5d38 Mon Sep 17 00:00:00 2001 From: Luca Natilla Date: Mon, 9 Jan 2023 12:15:05 +0100 Subject: [PATCH 38/62] Allow specifying files/directories to ignore when listing cached profiles. (#12845) * Allow specifying files/dirs to ignore when listing cached profiles. This is currently set to ignore just '.DS_Store' which was requested in the issue. To extend this behaviour it should be enough to modify `paths_to_ignore` in conan/api/subapi/profiles.py, as well as implementing the tests accordingly. * Remove unnecessary list conversion * Remove unnecessary complexity by applying suggested patches --- conan/api/subapi/profiles.py | 6 ++++++ conans/test/integration/command/test_profile.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/conan/api/subapi/profiles.py b/conan/api/subapi/profiles.py index 0a465bf5d07..e337b4ac51b 100644 --- a/conan/api/subapi/profiles.py +++ b/conan/api/subapi/profiles.py @@ -62,10 +62,16 @@ def list(self): List all the profiles file sin the cache :return: an alphabetically ordered list of profile files in the default cache location """ + # List is to be extended (directories should not have a trailing slash) + paths_to_ignore = ['.DS_Store'] + profiles = [] profiles_path = self._cache.profiles_path if os.path.exists(profiles_path): for current_directory, _, files in os.walk(profiles_path, followlinks=True): + files = filter(lambda file: os.path.relpath( + os.path.join(current_directory, file), profiles_path) not in paths_to_ignore, files) + for filename in files: rel_path = os.path.relpath(os.path.join(current_directory, filename), profiles_path) diff --git a/conans/test/integration/command/test_profile.py b/conans/test/integration/command/test_profile.py index e2be1330c24..0fc751e3ad9 100644 --- a/conans/test/integration/command/test_profile.py +++ b/conans/test/integration/command/test_profile.py @@ -1,3 +1,5 @@ +import os + from conans.test.utils.tools import TestClient @@ -5,3 +7,18 @@ def test_profile_path(): c = TestClient() c.run("profile path default") assert "default" in c.out + + +def test_ignore_paths_when_listing_profiles(): + c = TestClient() + ignore_path = '.DS_Store' + + # just in case + os.makedirs(c.cache.profiles_path, exist_ok=True) + # This a "touch" equivalent + open(os.path.join(c.cache.profiles_path, '.DS_Store'), 'w').close() + os.utime(os.path.join(c.cache.profiles_path, ".DS_Store")) + + c.run("profile list") + + assert ignore_path not in c.out From d0f21c07b814726c29943e1dc393afa0aaa4d798 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Jan 2023 12:43:20 +0100 Subject: [PATCH 39/62] new test for conf per package accumulation (#12862) * new test for conf per package accumulation * added build check --- .../configuration/conf/test_conf_profile.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/conans/test/integration/configuration/conf/test_conf_profile.py b/conans/test/integration/configuration/conf/test_conf_profile.py index c2c20866553..0f55ad7eb06 100644 --- a/conans/test/integration/configuration/conf/test_conf_profile.py +++ b/conans/test/integration/configuration/conf/test_conf_profile.py @@ -164,3 +164,36 @@ class Pkg(ConanFile): client.run("install . -pr=myprofile") msbuild_tool = client.load("conantoolchain_release_x64.props") assert "Async" in msbuild_tool + + +def test_config_package_append(client): + profile1 = textwrap.dedent("""\ + [conf] + user.myteam:myconf=["a", "b", "c"] + """) + profile2 = textwrap.dedent("""\ + include(profile1) + [conf] + mypkg*:user.myteam:myconf+=["d"] + mydep*:user.myteam:myconf=+["e"] + """) + conanfile = textwrap.dedent(""" + from conan import ConanFile + class Pkg(ConanFile): + def generate(self): + self.output.info(f"MYCONF: {self.conf.get('user.myteam:myconf')}") + def build(self): + self.output.info(f"MYCONFBUILD: {self.conf.get('user.myteam:myconf')}") + """) + client.save({"profile1": profile1, + "profile2": profile2, + "conanfile.py": conanfile}) + client.run("install . mypkg/0.1@ -pr=profile2") + assert "conanfile.py (mypkg/0.1): MYCONF: ['a', 'b', 'c', 'd']" in client.out + client.run("install . mydep/0.1@ -pr=profile2") + assert "conanfile.py (mydep/0.1): MYCONF: ['e', 'a', 'b', 'c']" in client.out + + client.run("create . mypkg/0.1@ -pr=profile2") + assert "mypkg/0.1: MYCONFBUILD: ['a', 'b', 'c', 'd']" in client.out + client.run("create . mydep/0.1@ -pr=profile2") + assert "mydep/0.1: MYCONFBUILD: ['e', 'a', 'b', 'c']" in client.out From 6e1151cd72df3c0d4b6ac7c046b980a665a4ce1c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Jan 2023 12:46:22 +0100 Subject: [PATCH 40/62] test with transitive python_requires_extend (#10568) * test with transitive python_requires_extend * new test --- .../py_requires/python_requires_test.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/conans/test/integration/py_requires/python_requires_test.py b/conans/test/integration/py_requires/python_requires_test.py index c2d5bab4052..174f92ad12e 100644 --- a/conans/test/integration/py_requires/python_requires_test.py +++ b/conans/test/integration/py_requires/python_requires_test.py @@ -1020,3 +1020,103 @@ def generate(self): c.run("install consumer") assert "VALUE COMMON: 42!!!" in c.out assert "VALUE TOOL: 42!!!" in c.out + + +class TestTransitiveExtend: + # https://github.com/conan-io/conan/issues/10511 + # https://github.com/conan-io/conan/issues/10565 + def test_transitive_extend(self): + client = TestClient() + company = textwrap.dedent(""" + from conan import ConanFile + class CompanyConanFile(ConanFile): + name = "company" + version = "1.0" + + def msg1(self): + return "company" + def msg2(self): + return "company" + """) + project = textwrap.dedent(""" + from conan import ConanFile + class ProjectBaseConanFile(ConanFile): + name = "project" + version = "1.0" + + python_requires = "company/1.0" + python_requires_extend = "company.CompanyConanFile" + + def msg1(self): + return "project" + """) + consumer = textwrap.dedent(""" + from conan import ConanFile + class Base(ConanFile): + name = "consumer" + version = "1.0" + python_requires = "project/1.0" + python_requires_extend = "project.ProjectBaseConanFile" + def generate(self): + self.output.info("Msg1:{}!!!".format(self.msg1())) + self.output.info("Msg2:{}!!!".format(self.msg2())) + """) + client.save({"company/conanfile.py": company, + "project/conanfile.py": project, + "consumer/conanfile.py": consumer}) + client.run("export company") + client.run("export project") + client.run("install consumer") + assert "conanfile.py (consumer/1.0): Msg1:project!!!" in client.out + assert "conanfile.py (consumer/1.0): Msg2:company!!!" in client.out + + def test_transitive_extend2(self): + client = TestClient() + company = textwrap.dedent(""" + from conan import ConanFile + class CompanyConanFile(ConanFile): + name = "company" + version = "1.0" + + class CompanyBase: + def msg1(self): + return "company" + def msg2(self): + return "company" + """) + project = textwrap.dedent(""" + from conan import ConanFile + class ProjectBase: + def msg1(self): + return "project" + + class ProjectBaseConanFile(ConanFile): + name = "project" + version = "1.0" + python_requires = "company/1.0" + + def init(self): + pkg_name, base_class_name = "company", "CompanyBase" + base_class = getattr(self.python_requires[pkg_name].module, base_class_name) + global ProjectBase + ProjectBase = type('ProjectBase', (ProjectBase, base_class, object), {}) + """) + consumer = textwrap.dedent(""" + from conan import ConanFile + class Base(ConanFile): + name = "consumer" + version = "1.0" + python_requires = "project/1.0" + python_requires_extend = "project.ProjectBase" + def generate(self): + self.output.info("Msg1:{}!!!".format(self.msg1())) + self.output.info("Msg2:{}!!!".format(self.msg2())) + """) + client.save({"company/conanfile.py": company, + "project/conanfile.py": project, + "consumer/conanfile.py": consumer}) + client.run("export company") + client.run("export project") + client.run("install consumer") + assert "conanfile.py (consumer/1.0): Msg1:project!!!" in client.out + assert "conanfile.py (consumer/1.0): Msg2:company!!!" in client.out From 7fa213bcd5984f0bb9d6d7897baf79ec0ee279f0 Mon Sep 17 00:00:00 2001 From: Gregor Jasny Date: Tue, 10 Jan 2023 12:52:14 +0100 Subject: [PATCH 41/62] Also write CMake Test Presets (#12846) * Also write CMake Test Presets * Refactor preset insertion logic --- conan/tools/cmake/presets.py | 37 +++++++++++------- .../toolchains/cmake/test_cmake_toolchain.py | 38 +++++++++++++++++++ .../toolchains/cmake/test_cmaketoolchain.py | 22 +++++++++++ 3 files changed, 84 insertions(+), 13 deletions(-) diff --git a/conan/tools/cmake/presets.py b/conan/tools/cmake/presets.py index dacf021effd..306b811d310 100644 --- a/conan/tools/cmake/presets.py +++ b/conan/tools/cmake/presets.py @@ -10,16 +10,24 @@ def _build_preset(conanfile, multiconfig): + return _common_build_and_test_preset_fields(conanfile, multiconfig) + + +def _test_preset(conanfile, multiconfig): + return _common_build_and_test_preset_fields(conanfile, multiconfig) + + +def _common_build_and_test_preset_fields(conanfile, multiconfig): build_type = conanfile.settings.get_safe("build_type") configure_preset_name = _configure_preset_name(conanfile, multiconfig) - ret = {"name": _build_preset_name(conanfile), + ret = {"name": _build_and_test_preset_name(conanfile), "configurePreset": configure_preset_name} if multiconfig: ret["configuration"] = build_type return ret -def _build_preset_name(conanfile): +def _build_and_test_preset_name(conanfile): build_type = conanfile.settings.get_safe("build_type") custom_conf, user_defined_build = get_build_folder_custom_vars(conanfile) if user_defined_build: @@ -108,6 +116,14 @@ def _format_val(val): return ret +def _insert_preset(data, preset_name, preset): + position = _get_already_existing_preset_index(preset["name"], data[preset_name]) + if position is not None: + data[preset_name][position] = preset + else: + data[preset_name].append(preset) + + def _forced_schema_2(conanfile): version = conanfile.conf.get("tools.cmake.cmaketoolchain.presets:max_schema_version", check_type=int) @@ -143,6 +159,7 @@ def _contents(conanfile, toolchain_file, cache_variables, generator): } multiconfig = is_multi_configuration(generator) ret["buildPresets"].append(_build_preset(conanfile, multiconfig)) + ret["testPresets"].append(_test_preset(conanfile, multiconfig)) _conf = _configure_preset(conanfile, generator, cache_variables, toolchain_file, multiconfig) ret["configurePresets"].append(_conf) return ret @@ -174,20 +191,14 @@ def write_cmake_presets(conanfile, toolchain_file, generator, cache_variables, if os.path.exists(preset_path): data = json.loads(load(preset_path)) build_preset = _build_preset(conanfile, multiconfig) - position = _get_already_existing_preset_index(build_preset["name"], data["buildPresets"]) - if position is not None: - data["buildPresets"][position] = build_preset - else: - data["buildPresets"].append(build_preset) + _insert_preset(data, "buildPresets", build_preset) + + test_preset = _test_preset(conanfile, multiconfig) + _insert_preset(data, "testPresets", test_preset) configure_preset = _configure_preset(conanfile, generator, cache_variables, toolchain_file, multiconfig) - position = _get_already_existing_preset_index(configure_preset["name"], - data["configurePresets"]) - if position is not None: - data["configurePresets"][position] = configure_preset - else: - data["configurePresets"].append(configure_preset) + _insert_preset(data, "configurePresets", configure_preset) else: data = _contents(conanfile, toolchain_file, cache_variables, generator) diff --git a/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py b/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py index dbd70681ef0..e172d3927fc 100644 --- a/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py +++ b/conans/test/functional/toolchains/cmake/test_cmake_toolchain.py @@ -594,9 +594,12 @@ def test_cmake_presets_multiple_settings_single_config(): presets = json.loads(load(user_presets["include"][0])) assert len(presets["configurePresets"]) == 1 assert len(presets["buildPresets"]) == 1 + assert len(presets["testPresets"]) == 1 assert presets["configurePresets"][0]["name"] == "apple-clang-12.0-gnu17-release" assert presets["buildPresets"][0]["name"] == "apple-clang-12.0-gnu17-release" assert presets["buildPresets"][0]["configurePreset"] == "apple-clang-12.0-gnu17-release" + assert presets["testPresets"][0]["name"] == "apple-clang-12.0-gnu17-release" + assert presets["testPresets"][0]["configurePreset"] == "apple-clang-12.0-gnu17-release" # If we create the "Debug" one, it has the same toolchain and preset file, that is # always multiconfig @@ -608,12 +611,17 @@ def test_cmake_presets_multiple_settings_single_config(): presets = json.loads(load(user_presets["include"][0])) assert len(presets["configurePresets"]) == 2 assert len(presets["buildPresets"]) == 2 + assert len(presets["testPresets"]) == 2 assert presets["configurePresets"][0]["name"] == "apple-clang-12.0-gnu17-release" assert presets["configurePresets"][1]["name"] == "apple-clang-12.0-gnu17-debug" assert presets["buildPresets"][0]["name"] == "apple-clang-12.0-gnu17-release" assert presets["buildPresets"][1]["name"] == "apple-clang-12.0-gnu17-debug" assert presets["buildPresets"][0]["configurePreset"] == "apple-clang-12.0-gnu17-release" assert presets["buildPresets"][1]["configurePreset"] == "apple-clang-12.0-gnu17-debug" + assert presets["testPresets"][0]["name"] == "apple-clang-12.0-gnu17-release" + assert presets["testPresets"][1]["name"] == "apple-clang-12.0-gnu17-debug" + assert presets["testPresets"][0]["configurePreset"] == "apple-clang-12.0-gnu17-release" + assert presets["testPresets"][1]["configurePreset"] == "apple-clang-12.0-gnu17-debug" # But If we change, for example, the cppstd and the compiler version, the toolchain # and presets will be different, but it will be appended to the UserPresets.json @@ -629,26 +637,32 @@ def test_cmake_presets_multiple_settings_single_config(): presets = json.loads(load(user_presets["include"][1])) assert len(presets["configurePresets"]) == 1 assert len(presets["buildPresets"]) == 1 + assert len(presets["testPresets"]) == 1 assert presets["configurePresets"][0]["name"] == "apple-clang-13-gnu20-release" assert presets["buildPresets"][0]["name"] == "apple-clang-13-gnu20-release" assert presets["buildPresets"][0]["configurePreset"] == "apple-clang-13-gnu20-release" + assert presets["testPresets"][0]["name"] == "apple-clang-13-gnu20-release" + assert presets["testPresets"][0]["configurePreset"] == "apple-clang-13-gnu20-release" # We can build with cmake manually if platform.system() == "Darwin": client.run_command("cmake . --preset apple-clang-12.0-gnu17-release") client.run_command("cmake --build --preset apple-clang-12.0-gnu17-release") + client.run_command("ctest --preset apple-clang-12.0-gnu17-release") client.run_command("./build/apple-clang-12.0-gnu17/Release/hello") assert "Hello World Release!" in client.out assert "__cplusplus2017" in client.out client.run_command("cmake . --preset apple-clang-12.0-gnu17-debug") client.run_command("cmake --build --preset apple-clang-12.0-gnu17-debug") + client.run_command("ctest --preset apple-clang-12.0-gnu17-debug") client.run_command("./build/apple-clang-12.0-gnu17/Debug/hello") assert "Hello World Debug!" in client.out assert "__cplusplus2017" in client.out client.run_command("cmake . --preset apple-clang-13-gnu20-release") client.run_command("cmake --build --preset apple-clang-13-gnu20-release") + client.run_command("ctest --preset apple-clang-13-gnu20-release") client.run_command("./build/apple-clang-13-gnu20/Release/hello") assert "Hello World Release!" in client.out assert "__cplusplus2020" in client.out @@ -672,6 +686,7 @@ def test_cmake_presets_duplicated_install(multiconfig): assert os.path.exists(presets_path) contents = json.loads(load(presets_path)) assert len(contents["buildPresets"]) == 1 + assert len(contents["testPresets"]) == 1 def test_remove_missing_presets(): @@ -736,6 +751,7 @@ def test_cmake_presets_options_single_config(): shared_str = "shared_true" if shared else "shared_false" client.run_command("cmake . --preset apple-clang-{}-release".format(shared_str)) client.run_command("cmake --build --preset apple-clang-{}-release".format(shared_str)) + client.run_command("ctest --preset apple-clang-{}-release".format(shared_str)) the_lib = "libhello.a" if not shared else "libhello.dylib" path = os.path.join(client.current_folder, "build", "apple-clang-{}".format(shared_str), "release", the_lib) @@ -763,9 +779,12 @@ def test_cmake_presets_multiple_settings_multi_config(): presets = json.loads(load(user_presets["include"][0])) assert len(presets["configurePresets"]) == 1 assert len(presets["buildPresets"]) == 1 + assert len(presets["testPresets"]) == 1 assert presets["configurePresets"][0]["name"] == "dynamic-14" assert presets["buildPresets"][0]["name"] == "dynamic-14-release" assert presets["buildPresets"][0]["configurePreset"] == "dynamic-14" + assert presets["testPresets"][0]["name"] == "dynamic-14-release" + assert presets["testPresets"][0]["configurePreset"] == "dynamic-14" # If we create the "Debug" one, it has the same toolchain and preset file, that is # always multiconfig @@ -777,11 +796,16 @@ def test_cmake_presets_multiple_settings_multi_config(): presets = json.loads(load(user_presets["include"][0])) assert len(presets["configurePresets"]) == 1 assert len(presets["buildPresets"]) == 2 + assert len(presets["testPresets"]) == 2 assert presets["configurePresets"][0]["name"] == "dynamic-14" assert presets["buildPresets"][0]["name"] == "dynamic-14-release" assert presets["buildPresets"][1]["name"] == "dynamic-14-debug" assert presets["buildPresets"][0]["configurePreset"] == "dynamic-14" assert presets["buildPresets"][1]["configurePreset"] == "dynamic-14" + assert presets["testPresets"][0]["name"] == "dynamic-14-release" + assert presets["testPresets"][1]["name"] == "dynamic-14-debug" + assert presets["testPresets"][0]["configurePreset"] == "dynamic-14" + assert presets["testPresets"][1]["configurePreset"] == "dynamic-14" # But If we change, for example, the cppstd and the compiler version, the toolchain # and presets will be different, but it will be appended to the UserPresets.json @@ -796,19 +820,24 @@ def test_cmake_presets_multiple_settings_multi_config(): presets = json.loads(load(user_presets["include"][1])) assert len(presets["configurePresets"]) == 1 assert len(presets["buildPresets"]) == 1 + assert len(presets["testPresets"]) == 1 assert presets["configurePresets"][0]["name"] == "static-17" assert presets["buildPresets"][0]["name"] == "static-17-release" assert presets["buildPresets"][0]["configurePreset"] == "static-17" + assert presets["testPresets"][0]["name"] == "static-17-release" + assert presets["testPresets"][0]["configurePreset"] == "static-17" # We can build with cmake manually client.run_command("cmake . --preset dynamic-14") client.run_command("cmake --build --preset dynamic-14-release") + client.run_command("ctest --preset dynamic-14-release") client.run_command("build\\dynamic-14\\Release\\hello") assert "Hello World Release!" in client.out assert "MSVC_LANG2014" in client.out client.run_command("cmake --build --preset dynamic-14-debug") + client.run_command("ctest --preset dynamic-14-debug") client.run_command("build\\dynamic-14\\Debug\\hello") assert "Hello World Debug!" in client.out assert "MSVC_LANG2014" in client.out @@ -816,6 +845,7 @@ def test_cmake_presets_multiple_settings_multi_config(): client.run_command("cmake . --preset static-17") client.run_command("cmake --build --preset static-17-release") + client.run_command("ctest --preset static-17-release") client.run_command("build\\static-17\\Release\\hello") assert "Hello World Release!" in client.out assert "MSVC_LANG2017" in client.out @@ -843,10 +873,12 @@ def test_user_presets_version2(): if platform.system() == "Windows": client.run_command("cmake . --preset 14") client.run_command("cmake --build --preset 14-release") + client.run_command("ctest --preset 14-release") client.run_command(r"build\14\Release\hello.exe") else: client.run_command("cmake . --preset 14-release") client.run_command("cmake --build --preset 14-release") + client.run_command("ctest --preset 14-release") client.run_command("./build/14/Release/hello") assert "Hello World Release!" in client.out @@ -859,10 +891,12 @@ def test_user_presets_version2(): if platform.system() == "Windows": client.run_command("cmake . --preset 17") client.run_command("cmake --build --preset 17-release") + client.run_command("ctest --preset 17-release") client.run_command(r"build\17\Release\hello.exe") else: client.run_command("cmake . --preset 17-release") client.run_command("cmake --build --preset 17-release") + client.run_command("ctest --preset 17-release") client.run_command("./build/17/Release/hello") assert "Hello World Release!" in client.out @@ -949,10 +983,12 @@ def test_cmake_presets_with_conanfile_txt(): if platform.system() != "Windows": c.run_command("cmake --preset debug") c.run_command("cmake --build --preset debug") + c.run_command("ctest --preset debug") c.run_command("./build/Debug/foo") else: c.run_command("cmake --preset default") c.run_command("cmake --build --preset debug") + c.run_command("ctest --preset debug") c.run_command("build\\Debug\\foo") assert "Hello World Debug!" in c.out @@ -960,9 +996,11 @@ def test_cmake_presets_with_conanfile_txt(): if platform.system() != "Windows": c.run_command("cmake --preset release") c.run_command("cmake --build --preset release") + c.run_command("ctest --preset release") c.run_command("./build/Release/foo") else: c.run_command("cmake --build --preset release") + c.run_command("ctest --preset release") c.run_command("build\\Release\\foo") assert "Hello World Release!" in c.out diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index 38a793f427a..535179ba6ff 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -401,12 +401,17 @@ def test_cmake_presets_multiconfig(): presets = json.loads(client.load("CMakePresets.json")) assert len(presets["buildPresets"]) == 1 assert presets["buildPresets"][0]["configuration"] == "Release" + assert len(presets["testPresets"]) == 1 + assert presets["testPresets"][0]["configuration"] == "Release" client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") presets = json.loads(client.load("CMakePresets.json")) assert len(presets["buildPresets"]) == 2 assert presets["buildPresets"][0]["configuration"] == "Release" assert presets["buildPresets"][1]["configuration"] == "Debug" + assert len(presets["testPresets"]) == 2 + assert presets["testPresets"][0]["configuration"] == "Release" + assert presets["testPresets"][1]["configuration"] == "Debug" client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=RelWithDebInfo " "--profile:h=profile") @@ -417,6 +422,11 @@ def test_cmake_presets_multiconfig(): assert presets["buildPresets"][1]["configuration"] == "Debug" assert presets["buildPresets"][2]["configuration"] == "RelWithDebInfo" assert presets["buildPresets"][3]["configuration"] == "MinSizeRel" + assert len(presets["testPresets"]) == 4 + assert presets["testPresets"][0]["configuration"] == "Release" + assert presets["testPresets"][1]["configuration"] == "Debug" + assert presets["testPresets"][2]["configuration"] == "RelWithDebInfo" + assert presets["testPresets"][3]["configuration"] == "MinSizeRel" # Repeat one client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") @@ -428,6 +438,12 @@ def test_cmake_presets_multiconfig(): assert presets["buildPresets"][2]["configuration"] == "RelWithDebInfo" assert presets["buildPresets"][3]["configuration"] == "MinSizeRel" + assert len(presets["testPresets"]) == 4 + assert presets["testPresets"][0]["configuration"] == "Release" + assert presets["testPresets"][1]["configuration"] == "Debug" + assert presets["testPresets"][2]["configuration"] == "RelWithDebInfo" + assert presets["testPresets"][3]["configuration"] == "MinSizeRel" + assert len(presets["configurePresets"]) == 1 assert presets["configurePresets"][0]["name"] == "default" @@ -453,6 +469,9 @@ def test_cmake_presets_singleconfig(): assert len(presets["buildPresets"]) == 1 assert presets["buildPresets"][0]["configurePreset"] == "release" + assert len(presets["testPresets"]) == 1 + assert presets["testPresets"][0]["configurePreset"] == "release" + # Now two configurePreset, but named correctly client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") presets = json.loads(client.load("CMakePresets.json")) @@ -462,6 +481,9 @@ def test_cmake_presets_singleconfig(): assert len(presets["buildPresets"]) == 2 assert presets["buildPresets"][1]["configurePreset"] == "debug" + assert len(presets["testPresets"]) == 2 + assert presets["testPresets"][1]["configurePreset"] == "debug" + # Repeat configuration, it shouldn't add a new one client.run("install mylib/1.0@ -g CMakeToolchain -s build_type=Debug --profile:h=profile") presets = json.loads(client.load("CMakePresets.json")) From cd6e7ea13dc070db47174af6e31bd85bae2bf9cd Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Jan 2023 13:28:59 +0100 Subject: [PATCH 42/62] new test to show build_folder_vars works in test_package (#12864) --- .../toolchains/cmake/test_cmaketoolchain.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index 535179ba6ff..dcbc046aa7f 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -898,6 +898,38 @@ def layout(self): c.run("install . -s os=Linux -s compiler=gcc -s compiler.version=7 -s build_type=Debug " "-s compiler.libcxx=libstdc++11 -s arch=x86 " "-c tools.cmake.cmake_layout:build_folder_vars='[\"settings.arch\", \"settings.build_type\"]'") - print(c.out) assert os.path.exists(os.path.join(c.current_folder, "build/x86-debug/generators/conan_toolchain.cmake")) + + +def test_test_package_layout(): + """ + test that the ``test_package`` folder also follows the cmake_layout and the + build_folder_vars + """ + client = TestClient() + test_conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.cmake import cmake_layout + + class Conan(ConanFile): + settings = "os", "arch", "compiler", "build_type" + generators = "CMakeToolchain" + test_type = "explicit" + + def requirements(self): + self.requires(self.tested_reference_str) + + def layout(self): + cmake_layout(self) + + def test(self): + pass + """) + client.save({"conanfile.py": GenConanfile("pkg", "0.1"), + "test_package/conanfile.py": test_conanfile}) + config = "-c tools.cmake.cmake_layout:build_folder_vars='[\"settings.compiler.cppstd\"]'" + client.run(f"create . {config} -s compiler.cppstd=14") + client.run(f"create . {config} -s compiler.cppstd=17") + assert os.path.exists(os.path.join(client.current_folder, "test_package/build/14")) + assert os.path.exists(os.path.join(client.current_folder, "test_package/build/17")) From 2794e39b90dbdd3aa1de33ec8e21de6795352d24 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Tue, 10 Jan 2023 20:35:21 +0700 Subject: [PATCH 43/62] Bugfix: fix KeyError 'buildPresets', the field is optional according to the specification (#12874) - fix KeyError 'buildPresets', the field is optional according to the specification Signed-off-by: SSE4 Signed-off-by: SSE4 --- conan/tools/cmake/presets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/tools/cmake/presets.py b/conan/tools/cmake/presets.py index 306b811d310..11a443a62f6 100644 --- a/conan/tools/cmake/presets.py +++ b/conan/tools/cmake/presets.py @@ -117,7 +117,7 @@ def _format_val(val): def _insert_preset(data, preset_name, preset): - position = _get_already_existing_preset_index(preset["name"], data[preset_name]) + position = _get_already_existing_preset_index(preset["name"], data.setdefault(preset_name, [])) if position is not None: data[preset_name][position] = preset else: From 04d97ce25af1ad6ae225deaf4a9c228c55857f1d Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Tue, 10 Jan 2023 16:23:51 +0100 Subject: [PATCH 44/62] Fix test_conan_test after layout changes (#12878) fix test --- conans/test/functional/command/test_command_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conans/test/functional/command/test_command_test.py b/conans/test/functional/command/test_command_test.py index 970e50aaf72..c34fddfff27 100644 --- a/conans/test/functional/command/test_command_test.py +++ b/conans/test/functional/command/test_command_test.py @@ -1,4 +1,5 @@ import os +import platform import time import unittest @@ -26,5 +27,6 @@ def test_conan_test(self): client.run("test test_package hello/0.1@lasote/stable -s hello:build_type=Debug " "--build missing") self.assertIn('hello/0.1: Hello World Debug!', client.out) + subfolder = "Release" if platform.system() != "Windows" else "" assert os.path.exists(os.path.join(client.current_folder, "test_package", - "build", "generators", "conaninfo.txt")) + "build", subfolder, "generators", "conaninfo.txt")) From c0746bd4580d8e3e489c09dc832813fd5245e086 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Jan 2023 17:23:47 +0100 Subject: [PATCH 45/62] fix remove_broken_symlinks (#12876) --- conan/tools/files/symlinks/symlinks.py | 4 ++-- conans/test/unittests/tools/files/test_symlinks.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conan/tools/files/symlinks/symlinks.py b/conan/tools/files/symlinks/symlinks.py index 98c4b2040a1..7700e8713b4 100644 --- a/conan/tools/files/symlinks/symlinks.py +++ b/conan/tools/files/symlinks/symlinks.py @@ -37,7 +37,7 @@ def remove_external_symlinks(conanfile, base_folder=None): for fullpath in get_symlinks(base_folder): link_target = os.readlink(fullpath) if not os.path.isabs(link_target): - link_target = os.path.join(base_folder, link_target) + link_target = os.path.join(os.path.dirname(fullpath), link_target) if not _path_inside(base_folder, link_target): os.unlink(fullpath) @@ -47,6 +47,6 @@ def remove_broken_symlinks(conanfile, base_folder=None): for fullpath in get_symlinks(base_folder): link_target = os.readlink(fullpath) if not os.path.isabs(link_target): - link_target = os.path.join(base_folder, link_target) + link_target = os.path.join(os.path.dirname(fullpath), link_target) if not os.path.exists(link_target): os.unlink(fullpath) diff --git a/conans/test/unittests/tools/files/test_symlinks.py b/conans/test/unittests/tools/files/test_symlinks.py index 83433228ab5..3a2fd174f70 100644 --- a/conans/test/unittests/tools/files/test_symlinks.py +++ b/conans/test/unittests/tools/files/test_symlinks.py @@ -17,7 +17,7 @@ def folders(): (os.path.join(tmp, "foo/var/file.txt"), "foo/var/other/absolute.txt"), # Absolute link (os.path.join(tmp, "foo/var"), "foo/var/other/other/myfolder"), # Absolute link folder (os.path.join(tmp, "foo/var/file.txt"), "foo/absolute.txt"), # Absolute link - ("foo/var/file.txt", "foo/var/other/relative.txt"), # Relative link + ("../file.txt", "foo/var/other/relative.txt"), # Relative link ("missing.txt", "foo/var/other/broken.txt"), # Broken link (outside_folder, "foo/var/other/absolute_outside"), # Absolute folder outside the folder ("../../../../../outside", "foo/absolute_outside"), # Relative folder outside the folder @@ -54,7 +54,7 @@ def test_absolute_to_relative_symlinks(folders): assert linked_to == "var/file.txt" linked_to = os.readlink(os.path.join(folder, "foo/var/other/relative.txt")).replace("\\", "/") - assert linked_to == "foo/var/file.txt" + assert linked_to == "../file.txt" linked_to = os.readlink(os.path.join(folder, "foo/var/other/broken.txt")) assert linked_to == "missing.txt" @@ -81,7 +81,7 @@ def test_remove_external_symlinks(folders): assert linked_to == os.path.join(folder, "foo/var/file.txt") linked_to = os.readlink(os.path.join(folder, "foo/var/other/relative.txt")) - assert linked_to == "foo/var/file.txt" + assert linked_to == "../file.txt" linked_to = os.readlink(os.path.join(folder, "foo/var/other/broken.txt")) assert linked_to == "missing.txt" @@ -112,7 +112,7 @@ def test_remove_broken_symlinks(folders): assert linked_to == os.path.join(folder, "foo/var/file.txt") linked_to = os.readlink(os.path.join(folder, "foo/var/other/relative.txt")) - assert linked_to == "foo/var/file.txt" + assert linked_to == "../file.txt" # This one is removed assert not os.path.islink(os.path.join(folder, "foo/var/other/broken.txt")) From f5e8eab918b10c8be5441062713eb563ed0c16d7 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 10 Jan 2023 17:26:43 +0100 Subject: [PATCH 46/62] test for path with special chars (#12863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test for path with special chars * Added test that fails with special chars in the pathname * Ops, I accidentally added a local change that I needed to back out of * making paths relative in virtualenvs scripts * keep linux as abs paths * wip Co-authored-by: Markus Rännare --- conan/tools/env/environment.py | 18 +++--- conans/client/generators/__init__.py | 8 ++- .../cache/test_home_special_char.py | 58 +++++++++++++++++++ .../env/test_virtualenv_powershell.py | 2 + 4 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 conans/test/integration/cache/test_home_special_char.py diff --git a/conan/tools/env/environment.py b/conan/tools/env/environment.py index edc229083a5..1745ecc5577 100644 --- a/conan/tools/env/environment.py +++ b/conan/tools/env/environment.py @@ -302,22 +302,22 @@ def apply(self): os.environ.update(old_env) def save_bat(self, file_location, generate_deactivate=True): - filepath, filename = os.path.split(file_location) - deactivate_file = os.path.join(filepath, "deactivate_{}".format(filename)) + _, filename = os.path.split(file_location) + deactivate_file = "deactivate_{}".format(filename) deactivate = textwrap.dedent("""\ setlocal - echo @echo off > "{deactivate_file}" - echo echo Restoring environment >> "{deactivate_file}" + echo @echo off > "%~dp0/{deactivate_file}" + echo echo Restoring environment >> "%~dp0/{deactivate_file}" for %%v in ({vars}) do ( set foundenvvar= for /f "delims== tokens=1,2" %%a in ('set') do ( if /I "%%a" == "%%v" ( - echo set "%%a=%%b">> "{deactivate_file}" + echo set "%%a=%%b">> "%~dp0/{deactivate_file}" set foundenvvar=1 ) ) if not defined foundenvvar ( - echo set %%v=>> "{deactivate_file}" + echo set %%v=>> "%~dp0/{deactivate_file}" ) ) endlocal @@ -335,9 +335,10 @@ def save_bat(self, file_location, generate_deactivate=True): save(file_location, content) def save_ps1(self, file_location, generate_deactivate=True,): - filepath, filename = os.path.split(file_location) - deactivate_file = os.path.join(filepath, "deactivate_{}".format(filename)) + _, filename = os.path.split(file_location) + deactivate_file = "deactivate_{}".format(filename) deactivate = textwrap.dedent("""\ + Push-Location $PSScriptRoot "echo `"Restoring environment`"" | Out-File -FilePath "{deactivate_file}" $vars = (Get-ChildItem env:*).name $updated_vars = @({vars}) @@ -354,6 +355,7 @@ def save_ps1(self, file_location, generate_deactivate=True,): Add-Content "{deactivate_file}" "`nif (Test-Path env:$var) {{ Remove-Item env:$var }}" }} }} + Pop-Location """).format( deactivate_file=deactivate_file, vars=",".join(['"{}"'.format(var) for var in self._values.keys()]) diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index dad8cdc361b..3413e98d017 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -291,12 +291,16 @@ def deactivates(filenames): ps1s = [] for env_script in env_scripts: path = os.path.join(conanfile.generators_folder, env_script) + # Only the .bat and .ps1 are made relative to current script if env_script.endswith(".bat"): - bats.append(path) + path = os.path.relpath(path, conanfile.generators_folder) + bats.append("%~dp0/"+path) elif env_script.endswith(".sh"): shs.append(subsystem_path(subsystem, path)) elif env_script.endswith(".ps1"): - ps1s.append(path) + path = os.path.relpath(path, conanfile.generators_folder) + # This $PSScriptRoot uses the current script directory + ps1s.append("$PSScriptRoot/"+path) if shs: def sh_content(files): return ". " + " && . ".join('"{}"'.format(s) for s in files) diff --git a/conans/test/integration/cache/test_home_special_char.py b/conans/test/integration/cache/test_home_special_char.py new file mode 100644 index 00000000000..d958edda3bb --- /dev/null +++ b/conans/test/integration/cache/test_home_special_char.py @@ -0,0 +1,58 @@ +import os +import platform + +from conans.test.utils.test_files import temp_folder +from conans.test.utils.tools import TestClient + +import textwrap + + +def test_home_special_chars(): + """ the path with special characters is creating a conanbuild.bat that fails + """ + path_chars = "päthñç$" + cache_folder = os.path.join(temp_folder(), path_chars) + current_folder = os.path.join(temp_folder(), path_chars) + c = TestClient(cache_folder, current_folder) + + tool = textwrap.dedent(r""" + import os + from conan import ConanFile + from conan.tools.files import save, chdir + class Pkg(ConanFile): + name = "mytool" + version = "1.0" + def package(self): + with chdir(self, self.package_folder): + echo = "@echo off\necho MYTOOL WORKS!!" + save(self, "bin/mytool.bat", echo) + save(self, "bin/mytool.sh", echo) + os.chmod("bin/mytool.sh", 0o777) + """) + c.save({"conanfile.py": tool}) + c.run("create .") + + conan_file = textwrap.dedent(""" + import platform + from conan import ConanFile + + class App(ConanFile): + name="failure" + version="0.1" + settings = 'os', 'arch', 'compiler', 'build_type' + generators = "VirtualBuildEnv" + tool_requires = "mytool/1.0" + def build(self): + mycmd = "mytool.bat" if platform.system() == "Windows" else "mytool.sh" + self.run(mycmd) + """) + + c.save({"conanfile.py": conan_file}) + # Need the 2 profile to work correctly buildenv + c.run("create . -s:b build_type=Release") + assert path_chars in c.out + assert "MYTOOL WORKS!!" in c.out + if platform.system() == "Windows": + c.run("create . -s:b build_type=Release -c tools.env.virtualenv:powershell=True") + assert path_chars in c.out + assert "MYTOOL WORKS!!" in c.out diff --git a/conans/test/integration/toolchains/env/test_virtualenv_powershell.py b/conans/test/integration/toolchains/env/test_virtualenv_powershell.py index 4aa80ef0ea8..00f36f7fa11 100644 --- a/conans/test/integration/toolchains/env/test_virtualenv_powershell.py +++ b/conans/test/integration/toolchains/env/test_virtualenv_powershell.py @@ -14,6 +14,7 @@ def client(): # We use special characters and spaces, to check everything works # https://github.com/conan-io/conan/issues/12648 + # FIXME: This path still fails the creation of the deactivation script cache_folder = os.path.join(temp_folder(), "[sub] folder") client = TestClient(cache_folder) conanfile = str(GenConanfile("pkg", "0.1")) @@ -40,6 +41,7 @@ class ConanFileToolsTest(ConanFile): name = "app" version = "0.1" requires = "pkg/0.1" + apply_env = False def build(self): self.output.info("----------BUILD----------------") From 0f73ed743ffd5d9cfeef54bfbc1547d08ce6436d Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 11 Jan 2023 08:57:25 +0000 Subject: [PATCH 47/62] AutootolsToolchain: pass a valid GNU triplet when the compiler is msvc (#12881) --- conan/tools/gnu/get_gnu_triplet.py | 4 +--- conans/test/unittests/tools/gnu/test_triplets.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conan/tools/gnu/get_gnu_triplet.py b/conan/tools/gnu/get_gnu_triplet.py index 951247a680b..6adcc66355a 100644 --- a/conan/tools/gnu/get_gnu_triplet.py +++ b/conan/tools/gnu/get_gnu_triplet.py @@ -67,10 +67,8 @@ def _get_gnu_triplet(os_, arch, compiler=None): # Calculate the OS if compiler == "gcc": windows_op = "w64-mingw32" - elif compiler == "msvc": - windows_op = "windows-msvc" else: - windows_op = "windows" + windows_op = "unknown-windows" op_system = {"Windows": windows_op, "Linux": "linux-gnu", diff --git a/conans/test/unittests/tools/gnu/test_triplets.py b/conans/test/unittests/tools/gnu/test_triplets.py index d4dd4c4ad50..3c97cb24bba 100644 --- a/conans/test/unittests/tools/gnu/test_triplets.py +++ b/conans/test/unittests/tools/gnu/test_triplets.py @@ -31,7 +31,9 @@ ["Android", "armv7", None, "arm-linux-androideabi"], ["Android", "armv7hf", None, "arm-linux-androideabi"], ["Android", "armv8", None, "aarch64-linux-android"], - ["Windows", "x86", "msvc", "i686-windows-msvc"], + ["Windows", "x86", "msvc", "i686-unknown-windows"], + ["Windows", "x86_64", "msvc", "x86_64-unknown-windows"], + ["Windows", "armv8", "msvc", "aarch64-unknown-windows"], ["Windows", "x86", "gcc", "i686-w64-mingw32"], ["Windows", "x86_64", "gcc", "x86_64-w64-mingw32"], ["Darwin", "x86_64", None, "x86_64-apple-darwin"], From f3fb6c41246669646243f0c7d83c2c444743d896 Mon Sep 17 00:00:00 2001 From: Jordan Williams Date: Wed, 11 Jan 2023 05:55:22 -0600 Subject: [PATCH 48/62] Add tools.build:linker_scripts conf variable (#12854) * Add linker_scripts conf variable Adds a conf variable for supplying linker scripts to the linker with `-T`. Includes support for CMakeToolchain, AutotoolsToolchain, and MesonToolchain. * Use single quotes for quoting in AutotoolsToolchain * Use list type for conf variable in LinkerScriptsBlock Co-authored-by: James --- conan/tools/cmake/toolchain/blocks.py | 15 ++++++++++ conan/tools/cmake/toolchain/toolchain.py | 3 +- conan/tools/gnu/autotoolstoolchain.py | 2 ++ conan/tools/meson/toolchain.py | 4 ++- conans/model/conf.py | 1 + .../toolchains/cmake/test_cmaketoolchain.py | 21 +++++++++++++ .../toolchains/gnu/test_autotoolstoolchain.py | 30 +++++++++++++++++++ .../toolchains/meson/test_mesontoolchain.py | 29 ++++++++++++++++++ .../tools/cmake/test_cmaketoolchain.py | 7 +++++ .../tools/gnu/autotoolschain_test.py | 14 +++++++++ 10 files changed, 124 insertions(+), 2 deletions(-) diff --git a/conan/tools/cmake/toolchain/blocks.py b/conan/tools/cmake/toolchain/blocks.py index 5846f009705..60d9e575916 100644 --- a/conan/tools/cmake/toolchain/blocks.py +++ b/conan/tools/cmake/toolchain/blocks.py @@ -215,6 +215,21 @@ def context(self): return {"arch_flag": arch_flag} +class LinkerScriptsBlock(Block): + template = textwrap.dedent(""" + string(APPEND CONAN_EXE_LINKER_FLAGS {{ linker_script_flags }}) + """) + + def context(self): + linker_scripts = self._conanfile.conf.get( + "tools.build:linker_scripts", check_type=list, default=[]) + if not linker_scripts: + return + linker_scripts = [linker_script.replace('\\', '/') for linker_script in linker_scripts] + linker_script_flags = ['-T"' + linker_script + '"' for linker_script in linker_scripts] + return {"linker_script_flags": " ".join(linker_script_flags)} + + class CppStdBlock(Block): template = textwrap.dedent(""" message(STATUS "Conan toolchain: C++ Standard {{ cppstd }} with extensions {{ cppstd_extensions }}") diff --git a/conan/tools/cmake/toolchain/toolchain.py b/conan/tools/cmake/toolchain/toolchain.py index 2a0e181f804..0bfb04bff6a 100644 --- a/conan/tools/cmake/toolchain/toolchain.py +++ b/conan/tools/cmake/toolchain/toolchain.py @@ -12,7 +12,7 @@ from conan.tools.cmake.toolchain.blocks import ToolchainBlocks, UserToolchain, GenericSystemBlock, \ AndroidSystemBlock, AppleSystemBlock, FPicBlock, ArchitectureBlock, GLibCXXBlock, VSRuntimeBlock, \ CppStdBlock, ParallelBlock, CMakeFlagsInitBlock, TryCompileBlock, FindFiles, PkgConfigBlock, \ - SkipRPath, SharedLibBock, OutputDirsBlock, ExtraFlagsBlock, CompilersBlock + SkipRPath, SharedLibBock, OutputDirsBlock, ExtraFlagsBlock, CompilersBlock, LinkerScriptsBlock from conan.tools.intel import IntelCC from conan.tools.microsoft import VCVars from conan.tools.microsoft.visual import vs_ide_version @@ -133,6 +133,7 @@ def __init__(self, conanfile, generator=None): ("apple_system", AppleSystemBlock), ("fpic", FPicBlock), ("arch_flags", ArchitectureBlock), + ("linker_scripts", LinkerScriptsBlock), ("libcxx", GLibCXXBlock), ("vs_runtime", VSRuntimeBlock), ("cppstd", CppStdBlock), diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index 58115a6072b..0a8914df494 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -113,6 +113,8 @@ def ldflags(self): check_type=list) conf_flags.extend(self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list)) + linker_scripts = self._conanfile.conf.get("tools.build:linker_scripts", default=[], check_type=list) + conf_flags.extend(["-T'" + linker_script + "'" for linker_script in linker_scripts]) ret = ret + apple_flags + conf_flags + self.build_type_link_flags + self.extra_ldflags return self._filter_list_empty_fields(ret) diff --git a/conan/tools/meson/toolchain.py b/conan/tools/meson/toolchain.py index 17f55324249..be0a535931f 100644 --- a/conan/tools/meson/toolchain.py +++ b/conan/tools/meson/toolchain.py @@ -291,10 +291,12 @@ def _get_extra_flags(self): cflags = self._conanfile.conf.get("tools.build:cflags", default=[], check_type=list) sharedlinkflags = self._conanfile.conf.get("tools.build:sharedlinkflags", default=[], check_type=list) exelinkflags = self._conanfile.conf.get("tools.build:exelinkflags", default=[], check_type=list) + linker_scripts = self._conanfile.conf.get("tools.build:linker_scripts", default=[], check_type=list) + linker_script_flags = ['-T"' + linker_script + '"' for linker_script in linker_scripts] return { "cxxflags": cxxflags, "cflags": cflags, - "ldflags": sharedlinkflags + exelinkflags + "ldflags": sharedlinkflags + exelinkflags + linker_script_flags } @staticmethod diff --git a/conans/model/conf.py b/conans/model/conf.py index 62f91da8d24..4ed113b614e 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -54,6 +54,7 @@ "tools.build:sharedlinkflags": "List of extra flags used by CMakeToolchain for CMAKE_SHARED_LINKER_FLAGS_INIT variable", "tools.build:exelinkflags": "List of extra flags used by CMakeToolchain for CMAKE_EXE_LINKER_FLAGS_INIT variable", "tools.build:compiler_executables": "Defines a Python dict-like with the compilers path to be used. Allowed keys {'c', 'cpp', 'cuda', 'objc', 'objcpp', 'rc', 'fortran', 'asm', 'hip', 'ispc'}", + "tools.build:linker_scripts": "List of linker script files to pass to the linker used by different toolchains like CMakeToolchain, AutotoolsToolchain, and MesonToolchain", "tools.microsoft.bash:subsystem": "Set subsystem to use for Windows. Possible values: 'msys2', 'msys', 'cygwin', 'wsl' and 'sfu'", "tools.microsoft.bash:path": "Path to the shell executable. Default: 'bash'", "tools.apple:sdk_path": "Path for the sdk location. This value will be passed as SDKROOT or -isysroot depending on the generator used", diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index dcbc046aa7f..7f2bac7fb99 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -902,6 +902,27 @@ def layout(self): "build/x86-debug/generators/conan_toolchain.cmake")) +def test_set_linker_scripts(): + profile = textwrap.dedent(r""" + [settings] + os=Windows + arch=x86_64 + compiler=clang + compiler.version=15 + compiler.libcxx=libstdc++11 + [conf] + tools.build:linker_scripts=["/usr/local/src/flash.ld", "C:\\local\\extra_data.ld"] + """) + client = TestClient(path_with_spaces=False) + conanfile = GenConanfile().with_settings("os", "arch", "compiler")\ + .with_generator("CMakeToolchain") + client.save({"conanfile.py": conanfile, + "profile": profile}) + client.run("install . -pr:b profile -pr:h profile") + toolchain = client.load("conan_toolchain.cmake") + assert 'string(APPEND CONAN_EXE_LINKER_FLAGS -T"/usr/local/src/flash.ld" -T"C:/local/extra_data.ld")' in toolchain + + def test_test_package_layout(): """ test that the ``test_package`` folder also follows the cmake_layout and the diff --git a/conans/test/integration/toolchains/gnu/test_autotoolstoolchain.py b/conans/test/integration/toolchains/gnu/test_autotoolstoolchain.py index 9c91d3c8ef1..4bc28f2fe57 100644 --- a/conans/test/integration/toolchains/gnu/test_autotoolstoolchain.py +++ b/conans/test/integration/toolchains/gnu/test_autotoolstoolchain.py @@ -45,6 +45,36 @@ def test_extra_flags_via_conf(): assert 'export LDFLAGS="$LDFLAGS --flag5 --flag6"' in toolchain +def test_linker_scripts_via_conf(): + os_ = platform.system() + os_ = "Macos" if os_ == "Darwin" else os_ + + profile = textwrap.dedent(""" + [settings] + os=%s + compiler=gcc + compiler.version=6 + compiler.libcxx=libstdc++11 + arch=armv8 + build_type=Release + + [conf] + tools.build:sharedlinkflags+=["--flag5"] + tools.build:exelinkflags+=["--flag6"] + tools.build:linker_scripts+=["/linker/scripts/flash.ld", "/linker/scripts/extra_data.ld"] + """ % os_) + client = TestClient() + conanfile = GenConanfile().with_settings("os", "arch", "compiler", "build_type")\ + .with_generator("AutotoolsToolchain") + client.save({"conanfile.py": conanfile, + "profile": profile}) + client.run("install . --profile:build=profile --profile:host=profile") + toolchain = client.load("conanautotoolstoolchain{}".format('.bat' if os_ == "Windows" else '.sh')) + if os_ == "Windows": + assert 'set "LDFLAGS=%LDFLAGS% --flag5 --flag6 -T\'/linker/scripts/flash.ld\' -T\'/linker/scripts/extra_data.ld\'"' in toolchain + else: + assert 'export LDFLAGS="$LDFLAGS --flag5 --flag6 -T\'/linker/scripts/flash.ld\' -T\'/linker/scripts/extra_data.ld\'"' in toolchain + def test_not_none_values(): conanfile = textwrap.dedent(""" diff --git a/conans/test/integration/toolchains/meson/test_mesontoolchain.py b/conans/test/integration/toolchains/meson/test_mesontoolchain.py index 7a1932fb9cc..d360c170634 100644 --- a/conans/test/integration/toolchains/meson/test_mesontoolchain.py +++ b/conans/test/integration/toolchains/meson/test_mesontoolchain.py @@ -92,6 +92,35 @@ def test_extra_flags_via_conf(): assert "cpp_link_args = ['-flag0', '-other=val', '-flag5', '-flag6']" in content +def test_linker_scripts_via_conf(): + profile = textwrap.dedent(""" + [settings] + os=Windows + arch=x86_64 + compiler=gcc + compiler.version=9 + compiler.cppstd=17 + compiler.libcxx=libstdc++ + build_type=Release + + [buildenv] + LDFLAGS=-flag0 -other=val + + [conf] + tools.build:sharedlinkflags+=["-flag5"] + tools.build:exelinkflags+=["-flag6"] + tools.build:linker_scripts+=["/linker/scripts/flash.ld", "/linker/scripts/extra_data.ld"] + """) + t = TestClient() + t.save({"conanfile.txt": "[generators]\nMesonToolchain", + "profile": profile}) + + t.run("install . -pr=profile") + content = t.load(MesonToolchain.native_filename) + assert "c_link_args = ['-flag0', '-other=val', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content + assert "cpp_link_args = ['-flag0', '-other=val', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content + + def test_correct_quotes(): profile = textwrap.dedent(""" [settings] diff --git a/conans/test/unittests/tools/cmake/test_cmaketoolchain.py b/conans/test/unittests/tools/cmake/test_cmaketoolchain.py index d66be8732e4..25776873e3d 100644 --- a/conans/test/unittests/tools/cmake/test_cmaketoolchain.py +++ b/conans/test/unittests/tools/cmake/test_cmaketoolchain.py @@ -505,3 +505,10 @@ def test_compilers_block(conanfile): content = toolchain.content for compiler, lang in cmake_mapping.items(): assert f'set(CMAKE_{lang}_COMPILER "path_to_{compiler}")' in content + + +def test_linker_scripts_block(conanfile): + conanfile.conf.define("tools.build:linker_scripts", ["path_to_first_linker_script", "path_to_second_linker_script"]) + toolchain = CMakeToolchain(conanfile) + content = toolchain.content + assert f'string(APPEND CONAN_EXE_LINKER_FLAGS -T"path_to_first_linker_script" -T"path_to_second_linker_script")' in content diff --git a/conans/test/unittests/tools/gnu/autotoolschain_test.py b/conans/test/unittests/tools/gnu/autotoolschain_test.py index 93b7d31eca0..b8f8a4d081a 100644 --- a/conans/test/unittests/tools/gnu/autotoolschain_test.py +++ b/conans/test/unittests/tools/gnu/autotoolschain_test.py @@ -125,3 +125,17 @@ def test_compilers_mapping(): env = autotoolschain.environment().vars(conanfile) for compiler, env_var in autotools_mapping.items(): assert env[env_var] == f"path_to_{compiler}" + + +def test_linker_scripts(): + conanfile = ConanFileMock() + conanfile.conf = Conf() + conanfile.conf.define("tools.build:linker_scripts", ["path_to_first_linker_script", "path_to_second_linker_script"]) + settings = MockSettings({"build_type": "Release", + "os": "Windows", + "arch": "x86_64"}) + conanfile.settings = settings + autotoolschain = AutotoolsToolchain(conanfile) + env = autotoolschain.environment().vars(conanfile) + assert "-T'path_to_first_linker_script'" in env["LDFLAGS"] + assert "-T'path_to_second_linker_script'" in env["LDFLAGS"] From 32224c8dbdc29f4b8c435b138e31543374c8d8ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 11 Jan 2023 13:58:31 +0100 Subject: [PATCH 49/62] Add throw flag for `check_min_vs` tool function (#12880) * Add thorw flag for check_min_vs tool function * Rename throw to raise_invalid --- conan/tools/microsoft/visual.py | 12 ++++-- .../tools/microsoft/test_check_min_vs.py | 43 +++++++++++++------ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/conan/tools/microsoft/visual.py b/conan/tools/microsoft/visual.py index 882b867e5d5..65895a29a0c 100644 --- a/conan/tools/microsoft/visual.py +++ b/conan/tools/microsoft/visual.py @@ -8,7 +8,7 @@ CONAN_VCVARS_FILE = "conanvcvars.bat" -def check_min_vs(conanfile, version): +def check_min_vs(conanfile, version, raise_invalid=True): """ this is a helper method to allow the migration of 1.X->2.0 and VisualStudio->msvc settings withoug breaking recipes The legacy "Visual Studio" with different toolset is not managed, not worth the complexity @@ -30,9 +30,13 @@ def check_min_vs(conanfile, version): compiler_version += ".{}".format(compiler_update) if compiler_version and Version(compiler_version) < version: - msg = "This package doesn't work with VS compiler version '{}'" \ - ", it requires at least '{}'".format(compiler_version, version) - raise ConanInvalidConfiguration(msg) + if raise_invalid: + msg = f"This package doesn't work with VS compiler version '{compiler_version}'" \ + f", it requires at least '{version}'" + raise ConanInvalidConfiguration(msg) + else: + return False + return True def msvc_version_to_vs_ide_version(version): diff --git a/conans/test/unittests/tools/microsoft/test_check_min_vs.py b/conans/test/unittests/tools/microsoft/test_check_min_vs.py index bca089093c8..4810a6a8119 100644 --- a/conans/test/unittests/tools/microsoft/test_check_min_vs.py +++ b/conans/test/unittests/tools/microsoft/test_check_min_vs.py @@ -7,6 +7,23 @@ class TestCheckMinVS: + parametrize_vars = "compiler,version,update,minimum" + valid_parametrize_values = [ + ("Visual Studio", "15", None, "191"), + ("Visual Studio", "16", None, "192"), + ("msvc", "193", None, "193"), + ("msvc", "193", None, "192"), + ("msvc", "193", "2", "193.1"), + ] + + invalid_parametrize_values = [ + ("Visual Studio", "15", None, "192"), + ("Visual Studio", "16", None, "193.1"), + ("msvc", "192", None, "193"), + ("msvc", "193", None, "193.1"), + ("msvc", "193", "1", "193.2"), + ] + @staticmethod def _create_conanfile(compiler, version, update=None): settings = MockSettings({"compiler": compiler, @@ -15,25 +32,23 @@ def _create_conanfile(compiler, version, update=None): conanfile = MockConanfile(settings) return conanfile - @pytest.mark.parametrize("compiler,version,update,minimum", [ - ("Visual Studio", "15", None, "191"), - ("Visual Studio", "16", None, "192"), - ("msvc", "193", None, "193"), - ("msvc", "193", None, "192"), - ("msvc", "193", "2", "193.1"), - ]) + @pytest.mark.parametrize(parametrize_vars, valid_parametrize_values) def test_valid(self, compiler, version, update, minimum): conanfile = self._create_conanfile(compiler, version, update) check_min_vs(conanfile, minimum) - @pytest.mark.parametrize("compiler,version,update,minimum", [ - ("Visual Studio", "15", None, "192"), - ("Visual Studio", "16", None, "193.1"), - ("msvc", "192", None, "193"), - ("msvc", "193", None, "193.1"), - ("msvc", "193", "1", "193.2"), - ]) + @pytest.mark.parametrize(parametrize_vars, valid_parametrize_values) + def test_valid_nothrows(self, compiler, version, update, minimum): + conanfile = self._create_conanfile(compiler, version, update) + assert check_min_vs(conanfile, minimum, raise_invalid=False) + + @pytest.mark.parametrize(parametrize_vars, invalid_parametrize_values) def test_invalid(self, compiler, version, update, minimum): conanfile = self._create_conanfile(compiler, version, update) with pytest.raises(ConanInvalidConfiguration): check_min_vs(conanfile, minimum) + + @pytest.mark.parametrize(parametrize_vars, invalid_parametrize_values) + def test_invalid_nothrows(self, compiler, version, update, minimum): + conanfile = self._create_conanfile(compiler, version, update) + assert not check_min_vs(conanfile, minimum, raise_invalid=False) From 690eca4353d766886045e8ed722881e7fa4fc116 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 11 Jan 2023 13:59:15 +0100 Subject: [PATCH 50/62] allow skip the generation of conanvcvars.bat (#12882) * allow skip the generation of conanvcvars.bat * review * add comment --- conan/tools/microsoft/visual.py | 5 ++++- conans/model/conf.py | 2 +- .../toolchains/microsoft/vcvars_test.py | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/conan/tools/microsoft/visual.py b/conan/tools/microsoft/visual.py index 65895a29a0c..5050ee8b895 100644 --- a/conan/tools/microsoft/visual.py +++ b/conan/tools/microsoft/visual.py @@ -76,6 +76,10 @@ def generate(self, scope="build"): if compiler not in ("Visual Studio", "msvc", "clang"): return + vs_install_path = conanfile.conf.get("tools.microsoft.msbuild:installation_path") + if vs_install_path == "": # Empty string means "disable" + return + if compiler == "clang": # The vcvars only needed for LLVM/Clang and VS ClangCL, who define runtime if not conanfile.settings.get_safe("compiler.runtime"): @@ -97,7 +101,6 @@ def generate(self, scope="build"): vcvars_ver = _vcvars_vers(conanfile, compiler, vs_version) vcvarsarch = vcvars_arch(conanfile) - vs_install_path = conanfile.conf.get("tools.microsoft.msbuild:installation_path") # The vs_install_path is like # C:\Program Files (x86)\Microsoft Visual Studio\2019\Community # C:\Program Files (x86)\Microsoft Visual Studio\2017\Community diff --git a/conans/model/conf.py b/conans/model/conf.py index 4ed113b614e..fcad44d9613 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -35,7 +35,7 @@ "tools.microsoft.msbuild:verbosity": "Verbosity level for MSBuild: 'Quiet', 'Minimal', 'Normal', 'Detailed', 'Diagnostic'", "tools.microsoft.msbuild:vs_version": "Defines the IDE version when using the new msvc compiler", "tools.microsoft.msbuild:max_cpu_count": "Argument for the /m when running msvc to build parallel projects", - "tools.microsoft.msbuild:installation_path": "VS install path, to avoid auto-detect via vswhere, like C:/Program Files (x86)/Microsoft Visual Studio/2019/Community", + "tools.microsoft.msbuild:installation_path": "VS install path, to avoid auto-detect via vswhere, like C:/Program Files (x86)/Microsoft Visual Studio/2019/Community. Use empty string to disable", "tools.microsoft.msbuilddeps:exclude_code_analysis": "Suppress MSBuild code analysis for patterns", "tools.microsoft.msbuildtoolchain:compile_options": "Dictionary with MSBuild compiler options", "tools.intel:installation_path": "Defines the Intel oneAPI installation root path", diff --git a/conans/test/integration/toolchains/microsoft/vcvars_test.py b/conans/test/integration/toolchains/microsoft/vcvars_test.py index 27db87bbdac..fb6779ab8e0 100644 --- a/conans/test/integration/toolchains/microsoft/vcvars_test.py +++ b/conans/test/integration/toolchains/microsoft/vcvars_test.py @@ -4,6 +4,7 @@ import pytest +from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient @@ -36,6 +37,23 @@ def generate(self): assert not os.path.exists(os.path.join(client.current_folder, "conanbuild.bat")) +@pytest.mark.skipif(platform.system() not in ["Windows"], reason="Requires Windows") +def test_vcvars_generator_skip(): + """ + tools.microsoft.msbuild:installation_path=disabled avoids creation of conanvcvars.bat + """ + client = TestClient() + client.save({"conanfile.py": GenConanfile().with_generator("VCVars") + .with_settings("os", "compiler", + "arch", "build_type"), + "profile": 'include(default)\n[conf]\ntools.microsoft.msbuild:installation_path='}) + + client.run('install . -c tools.microsoft.msbuild:installation_path=""') + assert not os.path.exists(os.path.join(client.current_folder, "conanvcvars.bat")) + client.run('install . -pr=profile') + assert not os.path.exists(os.path.join(client.current_folder, "conanvcvars.bat")) + + @pytest.mark.skipif(platform.system() not in ["Windows"], reason="Requires Windows") def test_vcvars_generator_string(): client = TestClient(path_with_spaces=False) From 8894507cc9e7d162db2e85873cb7fe3560a84e0c Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 11 Jan 2023 13:07:22 +0000 Subject: [PATCH 51/62] AutotoolsToolchain: pass valid GNU triplet when cross-building (#12870) Fix _get_gnu_triplet to return valid triplet for windows when compiler is Visual Studio --- conan/tools/gnu/get_gnu_triplet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conan/tools/gnu/get_gnu_triplet.py b/conan/tools/gnu/get_gnu_triplet.py index c082f3007aa..d25e8c5068c 100644 --- a/conan/tools/gnu/get_gnu_triplet.py +++ b/conan/tools/gnu/get_gnu_triplet.py @@ -68,7 +68,7 @@ def _get_gnu_triplet(os_, arch, compiler=None): if compiler == "gcc": windows_op = "w64-mingw32" elif compiler == "Visual Studio": - windows_op = "windows-msvc" + windows_op = "unknown-windows" else: windows_op = "windows" From cae326f25568509b0c56dcf33668491330cad039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Wed, 11 Jan 2023 14:10:31 +0100 Subject: [PATCH 52/62] Add test for empty confs (#12868) --- .../configuration/conf/test_conf.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/conans/test/integration/configuration/conf/test_conf.py b/conans/test/integration/configuration/conf/test_conf.py index 3a92ae39683..81adbb8e795 100644 --- a/conans/test/integration/configuration/conf/test_conf.py +++ b/conans/test/integration/configuration/conf/test_conf.py @@ -186,3 +186,43 @@ def test_jinja_global_conf(client): assert "user.mycompany:dist={}".format(distro.id()) in client.out else: assert "user.mycompany:dist=42" in client.out + + +def test_empty_conf_valid(): + tc = TestClient() + profile = textwrap.dedent(r""" + [conf] + user.unset= + """) + conanfile = textwrap.dedent(r""" + from conan import ConanFile + + class BasicConanfile(ConanFile): + name = "pkg" + version = "1.0" + + def generate(self): + self.output.warning(f'My unset conf variable is: "{self.conf.get("user.unset")}"') + self.output.warning(f'My unset conf is {"NOT" if self.conf.get("user.unset") == None else ""} set') + """) + tc.save({"conanfile.py": conanfile, "profile": profile}) + + tc.run("create .") + assert 'pkg/1.0: WARN: My unset conf is NOT set' in tc.out + + tc.run("create . -pr=profile") + assert 'pkg/1.0: WARN: My unset conf variable is: ""' in tc.out + assert 'pkg/1.0: WARN: My unset conf is set' in tc.out + + tc.run("create . -c user.unset=") + assert 'pkg/1.0: WARN: My unset conf variable is: ""' in tc.out + assert 'pkg/1.0: WARN: My unset conf is set' in tc.out + + tc.run('create . -c user.unset=""') + assert 'pkg/1.0: WARN: My unset conf variable is: ""' in tc.out + assert 'pkg/1.0: WARN: My unset conf is set' in tc.out + + # And ensure this actually works for the normal case, just in case + tc.run("create . -c user.unset=Hello") + assert 'pkg/1.0: WARN: My unset conf variable is: "Hello"' in tc.out + assert 'pkg/1.0: WARN: My unset conf is set' in tc.out From b0b6ab23c721d997063176cc079b5a389ce56e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Wed, 11 Jan 2023 14:25:08 +0100 Subject: [PATCH 53/62] [AutotoolsToolchain] Added helpers `update_xxxx_args` (#12884) * Fixed bad overwriting of configuration arguments * Added target calculation and let users configure them * Minor changes * Added update_xxxx_args to add/prune flags effortlessly * Typo * Improved helpers * Removed target triplet * Fixed test --- conan/tools/gnu/autotoolstoolchain.py | 83 +++++++++++++++---- .../tools/gnu/autotoolschain_test.py | 67 +++++++++++++++ 2 files changed, 134 insertions(+), 16 deletions(-) diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index 0a8914df494..61e159ea4fd 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -7,7 +7,7 @@ from conan.tools.env import Environment from conan.tools.files.files import save_toolchain_args from conan.tools.gnu.get_gnu_triplet import _get_gnu_triplet -from conan.tools.microsoft import VCVars, is_msvc, msvc_runtime_flag +from conan.tools.microsoft import VCVars, msvc_runtime_flag from conans.errors import ConanException from conans.tools import args_to_string @@ -18,10 +18,6 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self._namespace = namespace self._prefix = prefix - self.configure_args = self._default_configure_shared_flags() + self._default_configure_install_flags() - self.autoreconf_args = self._default_autoreconf_flags() - self.make_args = [] - # Flags self.extra_cxxflags = [] self.extra_cflags = [] @@ -55,12 +51,13 @@ def __init__(self, conanfile, namespace=None, prefix="/"): self.sysroot_flag = None if cross_building(self._conanfile): + # Host triplet os_build, arch_build, os_host, arch_host = get_cross_building_settings(self._conanfile) compiler = self._conanfile.settings.get_safe("compiler") if not self._host: self._host = _get_gnu_triplet(os_host, arch_host, compiler=compiler) + # Build triplet self._build = _get_gnu_triplet(os_build, arch_build, compiler=compiler) - # Apple Stuff if os_build == "Macos": sdk_path = apple_sdk_path(conanfile) @@ -74,6 +71,12 @@ def __init__(self, conanfile, namespace=None, prefix="/"): sysroot = sysroot.replace("\\", "/") if sysroot is not None else None self.sysroot_flag = "--sysroot {}".format(sysroot) if sysroot else None + self.configure_args = self._default_configure_shared_flags() + \ + self._default_configure_install_flags() + \ + self._get_triplets() + self.autoreconf_args = self._default_autoreconf_flags() + self.make_args = [] + check_using_build_profile(self._conanfile) def _get_msvc_runtime_flag(self): @@ -180,19 +183,67 @@ def _get_argument(argument_name, cppinfo_name): _get_argument("datarootdir", "resdirs")]) return [el for el in configure_install_flags if el] - def _default_autoreconf_flags(self): + @staticmethod + def _default_autoreconf_flags(): return ["--force", "--install"] + def _get_triplets(self): + triplets = [] + for flag, value in (("--host=", self._host), ("--build=", self._build), + ("--target=", self._target)): + if value: + triplets.append(f'{flag}{value}') + return triplets + + def update_configure_args(self, updated_flags): + """ + Helper to update/prune flags from ``self.configure_args``. + + :param updated_flags: ``dict`` with arguments as keys and their argument values. + Notice that if argument value is ``None``, this one will be pruned. + """ + self._update_flags("configure_args", updated_flags) + + def update_make_args(self, updated_flags): + """ + Helper to update/prune arguments from ``self.make_args``. + + :param updated_flags: ``dict`` with arguments as keys and their argument values. + Notice that if argument value is ``None``, this one will be pruned. + """ + self._update_flags("make_args", updated_flags) + + def update_autoreconf_args(self, updated_flags): + """ + Helper to update/prune arguments from ``self.autoreconf_args``. + + :param updated_flags: ``dict`` with arguments as keys and their argument values. + Notice that if argument value is ``None``, this one will be pruned. + """ + self._update_flags("autoreconf_args", updated_flags) + + # FIXME: Remove all these update_xxxx whenever xxxx_args are dicts or new ones replace them + def _update_flags(self, attr_name, updated_flags): + _new_flags = [] + self_args = getattr(self, attr_name) + for index, flag in enumerate(self_args): + flag_name = flag.split("=")[0] + if flag_name in updated_flags: + new_flag_value = updated_flags[flag_name] + # if {"build": None} is passed, then "--build=xxxx" will be pruned + if new_flag_value is None: + continue + elif not new_flag_value: + _new_flags.append(flag_name) + else: + _new_flags.append(f"{flag_name}={new_flag_value}") + else: + _new_flags.append(flag) + # Update the current ones + setattr(self, attr_name, _new_flags) + def generate_args(self): - configure_args = [] - configure_args.extend(self.configure_args) - user_args_str = args_to_string(self.configure_args) - for flag, var in (("host", self._host), ("build", self._build), ("target", self._target)): - if var and flag not in user_args_str: - configure_args.append('--{}={}'.format(flag, var)) - - args = {"configure_args": args_to_string(configure_args), + args = {"configure_args": args_to_string(self.configure_args), "make_args": args_to_string(self.make_args), "autoreconf_args": args_to_string(self.autoreconf_args)} - save_toolchain_args(args, namespace=self._namespace) diff --git a/conans/test/unittests/tools/gnu/autotoolschain_test.py b/conans/test/unittests/tools/gnu/autotoolschain_test.py index b8f8a4d081a..026acc933a0 100644 --- a/conans/test/unittests/tools/gnu/autotoolschain_test.py +++ b/conans/test/unittests/tools/gnu/autotoolschain_test.py @@ -1,9 +1,29 @@ +from unittest.mock import patch + import pytest from conan.tools.gnu import AutotoolsToolchain from conans.errors import ConanException from conans.model.conf import Conf from conans.test.utils.mocks import ConanFileMock, MockSettings +from conans.tools import args_to_string + + +@pytest.fixture() +def cross_building_conanfile(): + settings_build = MockSettings({"os": "Linux", + "arch": "x86_64", + "compiler": "gcc", + "compiler.version": "11", + "compiler.libcxx": "libstdc++", + "build_type": "Release"}) + settings_target = MockSettings({"os": "Android", "arch": "armv8"}) + settings = MockSettings({"os": "Emscripten", "arch": "wasm"}) + conanfile = ConanFileMock() + conanfile.settings = settings + conanfile.settings_build = settings_build + conanfile.settings_target = settings_target + return conanfile def test_get_gnu_triplet_for_cross_building(): @@ -139,3 +159,50 @@ def test_linker_scripts(): env = autotoolschain.environment().vars(conanfile) assert "-T'path_to_first_linker_script'" in env["LDFLAGS"] assert "-T'path_to_second_linker_script'" in env["LDFLAGS"] + + +@patch("conan.tools.gnu.autotoolstoolchain.save_toolchain_args") +def test_check_configure_args_overwriting_and_deletion(save_args, cross_building_conanfile): + # Issue: https://github.com/conan-io/conan/issues/12642 + at = AutotoolsToolchain(cross_building_conanfile) + at.configure_args.extend([ + "--with-cross-build=my_path", + "--something-host=my_host" + ]) + at.generate_args() + configure_args = save_args.call_args[0][0]['configure_args'] + assert "--build=x86_64-linux-gnu" in configure_args + assert "--host=wasm32-local-emscripten" in configure_args + assert "--with-cross-build=my_path" in configure_args + assert "--something-host=my_host" in configure_args + # https://github.com/conan-io/conan/issues/12431 + at.configure_args.remove("--build=x86_64-linux-gnu") + at.configure_args.remove("--host=wasm32-local-emscripten") + at.generate_args() + configure_args = save_args.call_args[0][0]['configure_args'] + assert "--build=x86_64-linux-gnu" not in configure_args # removed + assert "--host=wasm32-local-emscripten" not in configure_args # removed + assert "--with-cross-build=my_path" in configure_args + assert "--something-host=my_host" in configure_args + + +def test_update_or_prune_any_args(cross_building_conanfile): + at = AutotoolsToolchain(cross_building_conanfile) + at.configure_args.append("--enable-flag1=false") + at.make_args.append("--complex-flag=complex-value") + # Update configure_args + at.update_configure_args({"--prefix": "/my/other/prefix", + "--build": None, # prune value + "--enable-flag1": ""}) + new_configure_args = args_to_string(at.configure_args) + assert "--prefix=/my/other/prefix" in new_configure_args + assert "--build=" not in new_configure_args # pruned + assert "--enable-flag1" in new_configure_args # flag without value + # Update autoreconf_args + at.update_autoreconf_args({"--force": None}) + new_autoreconf_args = args_to_string(at.autoreconf_args) + assert "'--force" not in new_autoreconf_args + # Update make_args + at.update_make_args({"--complex-flag": "new-value"}) + new_make_args = args_to_string(at.make_args) + assert "--complex-flag=new-value" in new_make_args From a42412598f767df7b1b33926a350bcf5ad884290 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 11 Jan 2023 14:13:52 +0000 Subject: [PATCH 54/62] Add unix_path_package_info_legacy compatibility function (#12887) * Add unix_path_legacy_compat for compatibility * Add unix_path_package_info_legacy --- conan/tools/microsoft/__init__.py | 2 +- conan/tools/microsoft/subsystems.py | 6 +++++ .../tools/microsoft/test_subsystem.py | 27 +++++++++++++++---- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/conan/tools/microsoft/__init__.py b/conan/tools/microsoft/__init__.py index 79598b28e5f..9c0bb920d5f 100644 --- a/conan/tools/microsoft/__init__.py +++ b/conan/tools/microsoft/__init__.py @@ -1,7 +1,7 @@ from conan.tools.microsoft.layout import vs_layout from conan.tools.microsoft.msbuild import MSBuild from conan.tools.microsoft.msbuilddeps import MSBuildDeps -from conan.tools.microsoft.subsystems import unix_path +from conan.tools.microsoft.subsystems import unix_path, unix_path_package_info_legacy from conan.tools.microsoft.toolchain import MSBuildToolchain from conan.tools.microsoft.nmaketoolchain import NMakeToolchain from conan.tools.microsoft.nmakedeps import NMakeDeps diff --git a/conan/tools/microsoft/subsystems.py b/conan/tools/microsoft/subsystems.py index 55789e8ea81..5d1a59e3559 100644 --- a/conan/tools/microsoft/subsystems.py +++ b/conan/tools/microsoft/subsystems.py @@ -1,6 +1,12 @@ from conans.client.subsystems import deduce_subsystem, subsystem_path +from conans.client.tools.win import unix_path as unix_path_legacy_tools def unix_path(conanfile, path, scope="build"): subsystem = deduce_subsystem(conanfile, scope=scope) return subsystem_path(subsystem, path) + +def unix_path_package_info_legacy(conanfile, path, path_flavor=None): + # Call legacy implementation, which has different logic + # to autodeduce the subsystem type for the conversion. + return unix_path_legacy_tools(path, path_flavor) diff --git a/conans/test/unittests/tools/microsoft/test_subsystem.py b/conans/test/unittests/tools/microsoft/test_subsystem.py index 65e5423e8c6..0e010e23afb 100644 --- a/conans/test/unittests/tools/microsoft/test_subsystem.py +++ b/conans/test/unittests/tools/microsoft/test_subsystem.py @@ -1,19 +1,20 @@ +import mock import textwrap - import pytest -from conan.tools.microsoft import unix_path +from conan.tools.microsoft import unix_path, unix_path_package_info_legacy from conans.model.conf import ConfDefinition from conans.test.utils.mocks import MockSettings, ConanFileMock - -@pytest.mark.parametrize("subsystem, expected_path", [ +expected_results = [ ("msys2", '/c/path/to/stuff'), ("msys", '/c/path/to/stuff'), ("cygwin", '/cygdrive/c/path/to/stuff'), ("wsl", '/mnt/c/path/to/stuff'), ("sfu", '/dev/fs/C/path/to/stuff') -]) +] + +@pytest.mark.parametrize("subsystem, expected_path", expected_results) def test_unix_path(subsystem, expected_path): c = ConfDefinition() c.loads(textwrap.dedent("""\ @@ -29,3 +30,19 @@ def test_unix_path(subsystem, expected_path): path = unix_path(conanfile, "c:/path/to/stuff") assert expected_path == path + +@mock.patch("platform.system", mock.MagicMock(return_value='Windows')) +@pytest.mark.parametrize("subsystem, expected_path", expected_results) +def test_unix_path_package_info_legacy_windows(subsystem, expected_path): + test_path = "c:/path/to/stuff" + conanfile = ConanFileMock() + package_info_legacy_path = unix_path_package_info_legacy(conanfile, test_path, path_flavor=subsystem) + assert expected_path == package_info_legacy_path + +@mock.patch("platform.system", mock.MagicMock(return_value='Darwin')) +@pytest.mark.parametrize("subsystem, expected_path", expected_results) +def test_unix_path_package_info_legacy_not_windows(subsystem, expected_path): + test_path = "c:/path/to/stuff" + conanfile = ConanFileMock() + package_info_legacy_path = unix_path_package_info_legacy(conanfile, test_path, path_flavor=subsystem) + assert test_path == package_info_legacy_path \ No newline at end of file From ea235dc8a4719b71e13d3734855bff4600f62b06 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Jan 2023 15:52:19 +0100 Subject: [PATCH 55/62] Conan 1.57 --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index fda098f9d4c..5c842d25f7c 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -20,4 +20,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, REVISIONS] # Server is always with revisions DEFAULT_REVISION_V1 = "0" -__version__ = '1.57.0-dev' +__version__ = '1.57.0' From c1557344eb4f6918129034091ecfce7971e9fd13 Mon Sep 17 00:00:00 2001 From: czoido Date: Wed, 11 Jan 2023 15:53:42 +0100 Subject: [PATCH 56/62] Conan 1.58.0-dev --- conans/__init__.py | 2 +- conans/client/migrations_settings.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index fda098f9d4c..78a7fc41509 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -20,4 +20,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, REVISIONS] # Server is always with revisions DEFAULT_REVISION_V1 = "0" -__version__ = '1.57.0-dev' +__version__ = '1.58.0-dev' diff --git a/conans/client/migrations_settings.py b/conans/client/migrations_settings.py index bdb5c3311bb..811bc2d7063 100644 --- a/conans/client/migrations_settings.py +++ b/conans/client/migrations_settings.py @@ -4187,3 +4187,5 @@ settings_1_56_0 = settings_1_55_0 settings_1_57_0 = settings_1_56_0 + +settings_1_58_0 = settings_1_57_0 From 5421e0206b9ce9404ec1c36ffdfc233cf37fb243 Mon Sep 17 00:00:00 2001 From: Luis Caro Campos <3535649+jcar87@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:55:11 +0000 Subject: [PATCH 57/62] Add unix_path_package_info_legacy compatibility function (#12886) Add unix_path_package_info_legacy --- conan/tools/microsoft/__init__.py | 2 +- conan/tools/microsoft/subsystems.py | 7 +++++++ conans/test/unittests/tools/microsoft/test_subsystem.py | 8 ++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/conan/tools/microsoft/__init__.py b/conan/tools/microsoft/__init__.py index 79598b28e5f..9c0bb920d5f 100644 --- a/conan/tools/microsoft/__init__.py +++ b/conan/tools/microsoft/__init__.py @@ -1,7 +1,7 @@ from conan.tools.microsoft.layout import vs_layout from conan.tools.microsoft.msbuild import MSBuild from conan.tools.microsoft.msbuilddeps import MSBuildDeps -from conan.tools.microsoft.subsystems import unix_path +from conan.tools.microsoft.subsystems import unix_path, unix_path_package_info_legacy from conan.tools.microsoft.toolchain import MSBuildToolchain from conan.tools.microsoft.nmaketoolchain import NMakeToolchain from conan.tools.microsoft.nmakedeps import NMakeDeps diff --git a/conan/tools/microsoft/subsystems.py b/conan/tools/microsoft/subsystems.py index 55789e8ea81..62600065c3e 100644 --- a/conan/tools/microsoft/subsystems.py +++ b/conan/tools/microsoft/subsystems.py @@ -4,3 +4,10 @@ def unix_path(conanfile, path, scope="build"): subsystem = deduce_subsystem(conanfile, scope=scope) return subsystem_path(subsystem, path) + +def unix_path_package_info_legacy(conanfile, path, path_flavor=None): + message = f"The use of 'unix_path_legacy_compat' is deprecated in Conan 2.0 and does not " \ + f"perform path conversions. This is retained for compatibility with Conan 1.x " \ + f"and will be removed in a future version." + conanfile.output.warning(message) + return path diff --git a/conans/test/unittests/tools/microsoft/test_subsystem.py b/conans/test/unittests/tools/microsoft/test_subsystem.py index 65e5423e8c6..3af3cf3c860 100644 --- a/conans/test/unittests/tools/microsoft/test_subsystem.py +++ b/conans/test/unittests/tools/microsoft/test_subsystem.py @@ -2,7 +2,7 @@ import pytest -from conan.tools.microsoft import unix_path +from conan.tools.microsoft import unix_path, unix_path_package_info_legacy from conans.model.conf import ConfDefinition from conans.test.utils.mocks import MockSettings, ConanFileMock @@ -27,5 +27,9 @@ def test_unix_path(subsystem, expected_path): conanfile.settings = settings conanfile.settings_build = settings - path = unix_path(conanfile, "c:/path/to/stuff") + test_path = "c:/path/to/stuff" + path = unix_path(conanfile, test_path) assert expected_path == path + + package_info_legacy_path = unix_path_package_info_legacy(conanfile, test_path, path_flavor=subsystem) + assert package_info_legacy_path == test_path From 33aec0b46c5f7f6308d5020dc77949ee7afefa25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Ram=C3=ADrez?= Date: Wed, 11 Jan 2023 17:07:21 +0100 Subject: [PATCH 58/62] [AutotoolsToolchain] Improve `update_xxxxx_args` behavior (#12889) * Fixed bad overwriting of configuration arguments * Added target calculation and let users configure them * Minor changes * Added update_xxxx_args to add/prune flags effortlessly * Typo * Improved helpers * Removed target triplet * Fixed test * Now, add/update/remove args * Update conan/tools/gnu/autotoolstoolchain.py Co-authored-by: James * Update conan/tools/gnu/autotoolstoolchain.py Co-authored-by: James Co-authored-by: James --- conan/tools/gnu/autotoolstoolchain.py | 35 +++++++++++-------- .../tools/gnu/autotoolschain_test.py | 12 ++++--- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/conan/tools/gnu/autotoolstoolchain.py b/conan/tools/gnu/autotoolstoolchain.py index 61e159ea4fd..b870bd5b032 100644 --- a/conan/tools/gnu/autotoolstoolchain.py +++ b/conan/tools/gnu/autotoolstoolchain.py @@ -224,23 +224,28 @@ def update_autoreconf_args(self, updated_flags): # FIXME: Remove all these update_xxxx whenever xxxx_args are dicts or new ones replace them def _update_flags(self, attr_name, updated_flags): - _new_flags = [] - self_args = getattr(self, attr_name) - for index, flag in enumerate(self_args): - flag_name = flag.split("=")[0] - if flag_name in updated_flags: - new_flag_value = updated_flags[flag_name] - # if {"build": None} is passed, then "--build=xxxx" will be pruned - if new_flag_value is None: - continue - elif not new_flag_value: - _new_flags.append(flag_name) + + def _list_to_dict(flags): + ret = {} + for flag in flags: + # Only splitting if "=" is there + option = flag.split("=", 1) + if len(option) == 2: + ret[option[0]] = option[1] else: - _new_flags.append(f"{flag_name}={new_flag_value}") - else: - _new_flags.append(flag) + ret[option[0]] = "" + return ret + + def _dict_to_list(flags): + return [f"{k}={v}" if v else k for k, v in flags.items() if v is not None] + + self_args = getattr(self, attr_name) + # FIXME: if xxxxx_args -> dict-type at some point, all these lines could be removed + options = _list_to_dict(self_args) + # Add/update/remove the current xxxxx_args with the new flags given + options.update(updated_flags) # Update the current ones - setattr(self, attr_name, _new_flags) + setattr(self, attr_name, _dict_to_list(options)) def generate_args(self): args = {"configure_args": args_to_string(self.configure_args), diff --git a/conans/test/unittests/tools/gnu/autotoolschain_test.py b/conans/test/unittests/tools/gnu/autotoolschain_test.py index 026acc933a0..61a20034a2d 100644 --- a/conans/test/unittests/tools/gnu/autotoolschain_test.py +++ b/conans/test/unittests/tools/gnu/autotoolschain_test.py @@ -189,20 +189,22 @@ def test_check_configure_args_overwriting_and_deletion(save_args, cross_building def test_update_or_prune_any_args(cross_building_conanfile): at = AutotoolsToolchain(cross_building_conanfile) at.configure_args.append("--enable-flag1=false") - at.make_args.append("--complex-flag=complex-value") # Update configure_args at.update_configure_args({"--prefix": "/my/other/prefix", "--build": None, # prune value - "--enable-flag1": ""}) + "--enable-flag1": "", # without value + "-NEW-FLAG": "no" # new flag + }) new_configure_args = args_to_string(at.configure_args) assert "--prefix=/my/other/prefix" in new_configure_args assert "--build=" not in new_configure_args # pruned assert "--enable-flag1" in new_configure_args # flag without value + assert "-NEW-FLAG=no" in new_configure_args # new flag # Update autoreconf_args at.update_autoreconf_args({"--force": None}) new_autoreconf_args = args_to_string(at.autoreconf_args) assert "'--force" not in new_autoreconf_args - # Update make_args - at.update_make_args({"--complex-flag": "new-value"}) + # Add new value to make_args + at.update_make_args({"--new-complex-flag": "new-value"}) new_make_args = args_to_string(at.make_args) - assert "--complex-flag=new-value" in new_make_args + assert "--new-complex-flag=new-value" in new_make_args From 85fc403e91d89f0c91e7207ea6264d62f86eb2e5 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 12 Jan 2023 00:43:38 +0100 Subject: [PATCH 59/62] fixing after develop merge --- conans/test/integration/cache/test_home_special_char.py | 9 ++++++++- .../integration/configuration/conf/test_conf_profile.py | 8 ++++---- .../integration/toolchains/cmake/test_cmaketoolchain.py | 8 ++++++-- conans/test/unittests/tools/gnu/autotoolschain_test.py | 9 +++++---- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/conans/test/integration/cache/test_home_special_char.py b/conans/test/integration/cache/test_home_special_char.py index d958edda3bb..ed2ff72a3cf 100644 --- a/conans/test/integration/cache/test_home_special_char.py +++ b/conans/test/integration/cache/test_home_special_char.py @@ -1,12 +1,18 @@ import os import platform +import pytest + from conans.test.utils.test_files import temp_folder from conans.test.utils.tools import TestClient import textwrap +# FIXME: Deactivating the legacy in-memory environment with apply_env=True to leave the env-var +# script only, it fails in Windows because .bat files are not UTF-8. I have tried a few things +# (like changing chcp, or reading from file) but seems quite challenging +@pytest.mark.xfail(reason="Path with special chars in Windows is still failing because of .bat") def test_home_special_chars(): """ the path with special characters is creating a conanbuild.bat that fails """ @@ -42,6 +48,7 @@ class App(ConanFile): settings = 'os', 'arch', 'compiler', 'build_type' generators = "VirtualBuildEnv" tool_requires = "mytool/1.0" + apply_env = False # SUPER IMPORTANT, DO NOT REMOVE def build(self): mycmd = "mytool.bat" if platform.system() == "Windows" else "mytool.sh" self.run(mycmd) @@ -49,7 +56,7 @@ def build(self): c.save({"conanfile.py": conan_file}) # Need the 2 profile to work correctly buildenv - c.run("create . -s:b build_type=Release") + c.run("create .") assert path_chars in c.out assert "MYTOOL WORKS!!" in c.out if platform.system() == "Windows": diff --git a/conans/test/integration/configuration/conf/test_conf_profile.py b/conans/test/integration/configuration/conf/test_conf_profile.py index e46d5ea72da..e0268b9b470 100644 --- a/conans/test/integration/configuration/conf/test_conf_profile.py +++ b/conans/test/integration/configuration/conf/test_conf_profile.py @@ -294,12 +294,12 @@ def build(self): client.save({"profile1": profile1, "profile2": profile2, "conanfile.py": conanfile}) - client.run("install . mypkg/0.1@ -pr=profile2") + client.run("install . --name=mypkg --version=0.1 -pr=profile2") assert "conanfile.py (mypkg/0.1): MYCONF: ['a', 'b', 'c', 'd']" in client.out - client.run("install . mydep/0.1@ -pr=profile2") + client.run("install . --name=mydep --version=0.1 -pr=profile2") assert "conanfile.py (mydep/0.1): MYCONF: ['e', 'a', 'b', 'c']" in client.out - client.run("create . mypkg/0.1@ -pr=profile2") + client.run("create . --name=mypkg --version=0.1 -pr=profile2") assert "mypkg/0.1: MYCONFBUILD: ['a', 'b', 'c', 'd']" in client.out - client.run("create . mydep/0.1@ -pr=profile2") + client.run("create . --name=mydep --version=0.1 -pr=profile2") assert "mydep/0.1: MYCONFBUILD: ['e', 'a', 'b', 'c']" in client.out diff --git a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py index 2d917aef81b..d1f424c5057 100644 --- a/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py +++ b/conans/test/integration/toolchains/cmake/test_cmaketoolchain.py @@ -958,6 +958,10 @@ def test(self): "test_package/conanfile.py": test_conanfile}) config = "-c tools.cmake.cmake_layout:build_folder_vars='[\"settings.compiler.cppstd\"]'" client.run(f"create . {config} -s compiler.cppstd=14") + assert os.path.exists(os.path.join(client.current_folder, "test_package/test_output/build/14")) client.run(f"create . {config} -s compiler.cppstd=17") - assert os.path.exists(os.path.join(client.current_folder, "test_package/build/14")) - assert os.path.exists(os.path.join(client.current_folder, "test_package/build/17")) + assert os.path.exists(os.path.join(client.current_folder, "test_package/test_output/build/17")) + # FIXME: We need to review this, it is counter-intuitive that you define a layout and it is + # simply erased at every conan create invocation + assert not os.path.exists(os.path.join(client.current_folder, + "test_package/test_output/build/14")) diff --git a/conans/test/unittests/tools/gnu/autotoolschain_test.py b/conans/test/unittests/tools/gnu/autotoolschain_test.py index d687122624f..aa735f9565e 100644 --- a/conans/test/unittests/tools/gnu/autotoolschain_test.py +++ b/conans/test/unittests/tools/gnu/autotoolschain_test.py @@ -2,11 +2,11 @@ import pytest +from conan.tools.build import cmd_args_to_string from conan.tools.gnu import AutotoolsToolchain from conans.errors import ConanException from conans.model.conf import Conf from conans.test.utils.mocks import ConanFileMock, MockSettings -from conans.tools import args_to_string @pytest.fixture() @@ -155,6 +155,7 @@ def test_linker_scripts(): conanfile.conf.define("tools.build:linker_scripts", ["path_to_first_linker_script", "path_to_second_linker_script"]) settings = MockSettings({"build_type": "Release", "os": "Windows", + "compiler": "gcc", "arch": "x86_64"}) conanfile.settings = settings autotoolschain = AutotoolsToolchain(conanfile) @@ -196,15 +197,15 @@ def test_update_or_prune_any_args(cross_building_conanfile): at.update_configure_args({"--prefix": "/my/other/prefix", "--build": None, # prune value "--enable-flag1": ""}) - new_configure_args = args_to_string(at.configure_args) + new_configure_args = cmd_args_to_string(at.configure_args) assert "--prefix=/my/other/prefix" in new_configure_args assert "--build=" not in new_configure_args # pruned assert "--enable-flag1" in new_configure_args # flag without value # Update autoreconf_args at.update_autoreconf_args({"--force": None}) - new_autoreconf_args = args_to_string(at.autoreconf_args) + new_autoreconf_args = cmd_args_to_string(at.autoreconf_args) assert "'--force" not in new_autoreconf_args # Update make_args at.update_make_args({"--complex-flag": "new-value"}) - new_make_args = args_to_string(at.make_args) + new_make_args = cmd_args_to_string(at.make_args) assert "--complex-flag=new-value" in new_make_args From 2f92345f2ca20089b50ea8bf150cd42f0e43d037 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 12 Jan 2023 01:02:18 +0100 Subject: [PATCH 60/62] fixing tests after develop merge --- conans/test/integration/toolchains/meson/test_mesontoolchain.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/test/integration/toolchains/meson/test_mesontoolchain.py b/conans/test/integration/toolchains/meson/test_mesontoolchain.py index 0fa23de2c60..e857397c392 100644 --- a/conans/test/integration/toolchains/meson/test_mesontoolchain.py +++ b/conans/test/integration/toolchains/meson/test_mesontoolchain.py @@ -115,7 +115,7 @@ def test_linker_scripts_via_conf(): t.save({"conanfile.txt": "[generators]\nMesonToolchain", "profile": profile}) - t.run("install . -pr=profile") + t.run("install . -pr:b=profile -pr=profile") content = t.load(MesonToolchain.native_filename) assert "c_link_args = ['-flag0', '-other=val', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content assert "cpp_link_args = ['-flag0', '-other=val', '-flag5', '-flag6', '-T\"/linker/scripts/flash.ld\"', '-T\"/linker/scripts/extra_data.ld\"']" in content From 1614b122e0a64b02c30849d7e1cd781cc440fad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rub=C3=A9n=20Rinc=C3=B3n=20Blanco?= Date: Thu, 12 Jan 2023 10:05:17 +0100 Subject: [PATCH 61/62] Add requires for `conan new cmake_{exe,lib}` (#12875) * Make new cmake_{lib,exe} able to require things * Add forggoten changes * Fix bug and simplify test * Mark test as needing CMake * Fix import path, even though it works locally for me * Add TODO to remind us to remove the new test once the feature is used elsewhere * Math->Mathematics to fix Windows --- conan/api/subapi/new.py | 21 ++++++--- conan/cli/commands/new.py | 5 ++- conan/internal/api/new/cmake_exe.py | 23 +++++++++- conan/internal/api/new/cmake_lib.py | 50 +++++++++++++++++++--- conans/test/functional/command/new_test.py | 24 +++++++++++ 5 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 conans/test/functional/command/new_test.py diff --git a/conan/api/subapi/new.py b/conan/api/subapi/new.py index e102798b756..4080e054f4d 100644 --- a/conan/api/subapi/new.py +++ b/conan/api/subapi/new.py @@ -9,7 +9,6 @@ class NewAPI: - _NOT_TEMPLATES = "not_templates" # Filename containing filenames of files not to be rendered def __init__(self, conan_api): @@ -84,17 +83,29 @@ def _read_files(self, template_folder): return template_files, non_template_files - @staticmethod def render(template_files, definitions): result = {} name = definitions.get("name", "Pkg") definitions["conan_version"] = __version__ - definitions["package_name"] = name.replace("-", "_").replace("+", "_").replace(".", "_") + + def as_package_name(n): + return n.replace("-", "_").replace("+", "_").replace(".", "_") + + def as_name(ref): + ref = as_package_name(ref) + if '/' in ref: + ref = ref[0:ref.index('/')] + return ref + + definitions["package_name"] = as_package_name(name) definitions["as_iterable"] = lambda x: [x] if isinstance(x, str) else x + definitions["as_name"] = as_name for k, v in template_files.items(): - k = Template(k, keep_trailing_newline=True, undefined=StrictUndefined).render(**definitions) - v = Template(v, keep_trailing_newline=True, undefined=StrictUndefined).render(**definitions) + k = Template(k, keep_trailing_newline=True, undefined=StrictUndefined).render( + **definitions) + v = Template(v, keep_trailing_newline=True, undefined=StrictUndefined).render( + **definitions) if v: result[k] = v return result diff --git a/conan/cli/commands/new.py b/conan/cli/commands/new.py index e8c2011b975..8bfe01e6fd8 100644 --- a/conan/cli/commands/new.py +++ b/conan/cli/commands/new.py @@ -67,8 +67,9 @@ def get_template_vars(): ast = env.parse(templ_str) template_vars.extend(meta.find_undeclared_variables(ast)) - injected_vars = {"conan_version", "package_name"} - template_vars = list(set(template_vars) - injected_vars) + injected_vars = {"conan_version", "package_name", "as_iterable", "as_name"} + optional_vars = {"requires"} + template_vars = list(set(template_vars) - injected_vars - optional_vars) template_vars.sort() return template_vars diff --git a/conan/internal/api/new/cmake_exe.py b/conan/internal/api/new/cmake_exe.py index aab9b0af83b..a5a57aa0e25 100644 --- a/conan/internal/api/new/cmake_exe.py +++ b/conan/internal/api/new/cmake_exe.py @@ -1,7 +1,7 @@ from conan.internal.api.new.cmake_lib import source_cpp, source_h, test_main conanfile_exe = '''from conan import ConanFile -from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout +from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps class {{package_name}}Recipe(ConanFile): @@ -26,6 +26,8 @@ def layout(self): cmake_layout(self) def generate(self): + deps = CMakeDeps(self) + deps.generate() tc = CMakeToolchain(self) tc.generate() @@ -37,13 +39,32 @@ def build(self): def package(self): cmake = CMake(self) cmake.install() + + {% if requires is defined -%} + def requirements(self): + {% for require in as_iterable(requires) -%} + self.requires("{{ require }}") + {% endfor %} + {%- endif %} ''' cmake_exe_v2 = """cmake_minimum_required(VERSION 3.15) project({{name}} CXX) +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +find_package({{as_name(require)}} CONFIG REQUIRED) +{% endfor %} +{%- endif %} + add_executable({{name}} src/{{name}}.cpp src/main.cpp) +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +target_link_libraries({{name}} PRIVATE {{as_name(require)}}::{{as_name(require)}}) +{% endfor %} +{%- endif %} + install(TARGETS {{name}} DESTINATION "." RUNTIME DESTINATION bin ARCHIVE DESTINATION lib diff --git a/conan/internal/api/new/cmake_lib.py b/conan/internal/api/new/cmake_lib.py index e1c404df0a3..f9e6bb2769e 100644 --- a/conan/internal/api/new/cmake_lib.py +++ b/conan/internal/api/new/cmake_lib.py @@ -1,5 +1,5 @@ conanfile_sources_v2 = '''from conan import ConanFile -from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout +from conan.tools.cmake import CMakeToolchain, CMake, cmake_layout, CMakeDeps class {{package_name}}Recipe(ConanFile): @@ -29,6 +29,8 @@ def layout(self): cmake_layout(self) def generate(self): + deps = CMakeDeps(self) + deps.generate() tc = CMakeToolchain(self) tc.generate() @@ -43,34 +45,66 @@ def package(self): def package_info(self): self.cpp_info.libs = ["{{name}}"] + + {% if requires is defined -%} + def requirements(self): + {% for require in as_iterable(requires) -%} + self.requires("{{ require }}") + {% endfor %} + {%- endif %} ''' cmake_v2 = """cmake_minimum_required(VERSION 3.15) project({{name}} CXX) +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +find_package({{as_name(require)}} CONFIG REQUIRED) +{% endfor %} +{%- endif %} + + add_library({{name}} src/{{name}}.cpp) target_include_directories({{name}} PUBLIC include) +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +target_link_libraries({{name}} PRIVATE {{as_name(require)}}::{{as_name(require)}}) +{% endfor %} +{%- endif %} + set_target_properties({{name}} PROPERTIES PUBLIC_HEADER "include/{{name}}.h") install(TARGETS {{name}}) """ source_h = """#pragma once -{% set define_name = name.replace("-", "_").replace("+", "_").replace(".", "_").upper() %} +{% set define_name = package_name.upper() %} #ifdef _WIN32 #define {{define_name}}_EXPORT __declspec(dllexport) #else #define {{define_name}}_EXPORT #endif -{{define_name}}_EXPORT void {{name.replace("-", "_").replace("+", "_").replace(".", "_")}}(); +{{define_name}}_EXPORT void {{package_name}}(); """ source_cpp = r"""#include #include "{{name}}.h" +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +#include "{{ as_name(require) }}.h" +{% endfor %} +{%- endif %} + + +void {{package_name}}(){ + {% if requires is defined -%} + {% for require in as_iterable(requires) -%} + {{ as_name(require) }}(); + {% endfor %} + {%- endif %} -void {{name.replace("-", "_").replace("+", "_").replace(".", "_")}}(){ #ifdef NDEBUG std::cout << "{{name}}/{{version}}: Hello World Release!\n"; #else @@ -198,6 +232,12 @@ def test(self): find_package({{name}} CONFIG REQUIRED) +{% if requires is defined -%} +{% for require in as_iterable(requires) -%} +find_package({{as_name(require)}} CONFIG REQUIRED) +{% endfor %} +{%- endif %} + add_executable(example src/example.cpp) target_link_libraries(example {{name}}::{{name}}) """ @@ -206,7 +246,7 @@ def test(self): test_main = """#include "{{name}}.h" int main() { - {{name.replace("-", "_").replace("+", "_").replace(".", "_")}}(); + {{package_name}}(); } """ diff --git a/conans/test/functional/command/new_test.py b/conans/test/functional/command/new_test.py new file mode 100644 index 00000000000..a1a10952e1d --- /dev/null +++ b/conans/test/functional/command/new_test.py @@ -0,0 +1,24 @@ +import pytest as pytest + +from conans.test.utils.tools import TestClient + + +# TODO: Remove this test once this feature is used elsewhere to test other things +@pytest.mark.tool("cmake") +def test_diamonds_cmake_with_new(): + tc = TestClient() + tc.run("new cmake_lib -d name=mathematics -d version=3.14") + tc.run("create .") + tc.run("new cmake_lib -d name=ai -d version=0.1 -d requires=mathematics/3.14 -f") + tc.run("create .") + assert "mathematics/3.14: Hello World" in tc.out + tc.run("new cmake_lib -d name=ui -d version=1.0 -d requires=mathematics/3.14 -f") + tc.run("create .") + assert "mathematics/3.14: Hello World" in tc.out + tc.run("new cmake_exe -d name=game -d version=0.0 -d requires=ai/0.1 -d requires=ui/1.0 -f") + tc.run("create .") + assert "mathematics/3.14: Hello World" in tc.out + assert "ai/0.1: Hello World" in tc.out + assert "ui/1.0: Hello World" in tc.out + + From 4f2f8bbddebf04b500076edcb5a207a5d3d8f1ed Mon Sep 17 00:00:00 2001 From: James Date: Thu, 12 Jan 2023 12:36:12 +0100 Subject: [PATCH 62/62] [develop2] New serialization json and printing for ``conan list`` (#12883) * [develop2] tests list * wip * wip * wip * fix tests * wip * wip * fix test --- conan/api/model.py | 39 +- conan/api/subapi/list.py | 17 +- conan/cli/commands/list.py | 114 +- conan/cli/commands/search.py | 3 +- .../build_requires/test_toolchain_packages.py | 3 +- .../test/integration/command/remove_test.py | 8 +- .../test/integration/command_v2/list_test.py | 1069 ++++++++--------- .../integration/command_v2/search_test.py | 22 +- .../remote/test_conaninfo_parsing.py | 4 +- 9 files changed, 623 insertions(+), 656 deletions(-) diff --git a/conan/api/model.py b/conan/api/model.py index 874d6219f60..2f1a43b3b55 100644 --- a/conan/api/model.py +++ b/conan/api/model.py @@ -1,5 +1,7 @@ from collections import OrderedDict +from conans.util.dates import timestamp_to_str + class Remote: @@ -81,26 +83,31 @@ def prefs(self): return prefs def add_prefs(self, prefs, configurations=None): - confs = configurations or OrderedDict() for pref in prefs: - binary_info = confs.get(pref, OrderedDict()) + binary_info = configurations.get(pref) if configurations is not None else None self.recipes.setdefault(pref.ref, []).append((pref, binary_info)) - @property - def ordered_recipes_by_name(self): - ret = OrderedDict() - for ref, prefs in self.recipes.items(): - ret.setdefault(ref.name, OrderedDict()).setdefault(ref, prefs) - return ret - def serialize(self): - ret = OrderedDict() - for ref, prefs in self.recipes.items(): - pref_ret = OrderedDict() - if prefs: - for pref, binary_info in prefs: - pref_ret[pref.repr_notime()] = binary_info - ret[ref.repr_notime()] = pref_ret + ret = {} + for ref, prefs in sorted(self.recipes.items()): + ref_dict = ret.setdefault(str(ref), {}) + if ref.revision: + revisions_dict = ref_dict.setdefault("revisions", {}) + rev_dict = revisions_dict.setdefault(ref.revision, {}) + if ref.timestamp: + rev_dict["timestamp"] = timestamp_to_str(ref.timestamp) + if prefs: + packages_dict = rev_dict.setdefault("packages", {}) + for pref_info in prefs: + pref, info = pref_info + pid_dict = packages_dict.setdefault(pref.package_id, {}) + if info is not None: + pid_dict["info"] = info + if pref.revision: + prevs_dict = pid_dict.setdefault("revisions", {}) + prev_dict = prevs_dict.setdefault(pref.revision, {}) + if pref.timestamp: + prev_dict["timestamp"] = timestamp_to_str(pref.timestamp) return ret diff --git a/conan/api/subapi/list.py b/conan/api/subapi/list.py index e78afdfccc8..d87cb048724 100644 --- a/conan/api/subapi/list.py +++ b/conan/api/subapi/list.py @@ -3,7 +3,7 @@ from conan.api.model import Remote, SelectBundle from conan.internal.api.select_pattern import ListPatternMode from conan.internal.conan_app import ConanApp -from conans.errors import ConanException +from conans.errors import ConanException, NotFoundException from conans.model.package_ref import PkgReference from conans.model.recipe_ref import RecipeReference from conans.search.search import get_cache_packages_binary_info, filter_packages @@ -38,13 +38,16 @@ def recipe_revisions(self, ref: RecipeReference, remote=None): return results def latest_package_revision(self, pref: PkgReference, remote=None): + # TODO: This returns None if the given package_id is not existing. It should probably + # raise NotFound, but to keep aligned with the above ``latest_recipe_revision`` which + # is used as an "exists" check too in other places, lets respect the None return assert pref.revision is None, "latest_package_revision: ref already have a revision" + assert pref.package_id is not None, "package_id must be defined" app = ConanApp(self.conan_api.cache_folder) if remote: ret = app.remote_manager.get_latest_package_reference(pref, remote=remote) else: ret = app.cache.get_latest_package_reference(pref) - return ret def package_revisions(self, pref: PkgReference, remote: Remote=None): @@ -116,7 +119,7 @@ def select(self, pattern, package_query=None, remote=None, search_mode=None): continue for rrev in rrevs: - packages = {} + packages = None prefs = [] if pattern.package_id and "*" not in pattern.package_id: prefs.append(PkgReference(rrev, package_id=pattern.package_id)) @@ -132,6 +135,9 @@ def select(self, pattern, package_query=None, remote=None, search_mode=None): # Show all the package IDs and their configurations if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: # add pref and its package configuration + # remove timestamp, as server does not provide it + for p in prefs: + p.timestamp = None select_bundle.add_prefs(prefs, configurations=packages) continue @@ -139,7 +145,10 @@ def select(self, pattern, package_query=None, remote=None, search_mode=None): if search_mode in (ListPatternMode.SHOW_LATEST_PREV, ListPatternMode.SHOW_ALL_PREVS): if pattern.is_latest_prev: - prevs = [self.conan_api.list.latest_package_revision(pref, remote)] + prev = self.conan_api.list.latest_package_revision(pref, remote) + if prev is None: + raise NotFoundException(f"Binary package not found: '{pref}") + prevs = [prev] else: prevs = self.conan_api.list.package_revisions(pref, remote) prevs = pattern.filter_prevs(prevs) diff --git a/conan/cli/commands/list.py b/conan/cli/commands/list.py index 643e8ef3256..81f04be78c7 100644 --- a/conan/cli/commands/list.py +++ b/conan/cli/commands/list.py @@ -3,11 +3,10 @@ from conan.api.conan_api import ConanAPI from conan.api.output import Color, cli_out_write from conan.cli.command import conan_command, OnceArgument -from conan.cli.commands import ConanJSONEncoder from conan.cli.formatters.list import list_packages_html -from conan.internal.api.select_pattern import ListPattern, ListPatternMode -from conans.util.dates import timestamp_to_str +from conan.internal.api.select_pattern import ListPattern +# Keep them so we don't break other commands that import them, but TODO: Remove later remote_color = Color.BRIGHT_BLUE recipe_name_color = Color.GREEN recipe_color = Color.BRIGHT_WHITE @@ -17,65 +16,71 @@ value_color = Color.CYAN +def print_serial(item, indent=None, color_index=None): + indent = "" if indent is None else (indent + " ") + color_index = 0 if color_index is None else (color_index + 1) + color_array = [Color.BRIGHT_BLUE, Color.GREEN, Color.BRIGHT_WHITE, + Color.BRIGHT_YELLOW, Color.CYAN, Color.WHITE] + color = color_array[color_index % len(color_array)] + if isinstance(item, dict): + for k, v in item.items(): + if isinstance(v, str): + if k.lower() == "error": + color = Color.BRIGHT_RED + k = "ERROR" + elif k.lower() == "warning": + color = Color.BRIGHT_YELLOW + k = "WARN" + cli_out_write(f"{indent}{k}: {v}", fg=color) + else: + cli_out_write(f"{indent}{k}", fg=color) + print_serial(v, indent, color_index) + elif isinstance(item, type([])): + for elem in item: + cli_out_write(f"{indent}{elem}", fg=color) + elif item: + cli_out_write(f"{indent}{item}", fg=color) + + def print_list_text(results): + """ Do litte format modification to serialized + list bundle so it looks prettier on text output + """ info = results["results"] - search_mode = results["search_mode"] - indentation = " " - - for remote, info_per_ref_name in info.items(): - cli_out_write(f"{remote}:", fg=remote_color) - if info_per_ref_name.get("error"): - cli_out_write(f" ERROR: {info_per_ref_name.get('error')}", fg=error_color) - continue + # Extract command single package name + new_info = {} + for remote, remote_info in info.items(): + new_remote_info = {} + for ref, content in remote_info.items(): + if ref == "error": + new_remote_info[ref] = content + else: + name, _ = ref.split("/", 1) + new_remote_info.setdefault(name, {})[ref] = content + new_info[remote] = new_remote_info + info = new_info - if not info_per_ref_name: - cli_out_write(f" There are no matching recipe references", fg=recipe_color) - continue + info = {remote: {"warning": "There are no matching recipe references"} if not values else values + for remote, values in info.items()} - for ref_name, refs in info_per_ref_name.items(): - cli_out_write(f"{indentation}{ref_name}", fg=recipe_name_color) - for ref, prefs in refs.items(): - cli_out_write(f"{indentation * 2}{ref.repr_humantime() if ref.timestamp else ref}", - fg=recipe_color) - if prefs: - for pref, binary_info in prefs: - pref_date = f" ({timestamp_to_str(pref.timestamp)})" if pref.timestamp \ - else "" - if search_mode == ListPatternMode.SHOW_PACKAGE_IDS: - cli_out_write(f"{indentation * 3}PID: {pref.package_id}{pref_date}", - fg=reference_color) - if not binary_info: - cli_out_write(f"{indentation * 4}Empty package information", - fg=field_color) - continue - elif search_mode in (ListPatternMode.SHOW_ALL_PREVS, - ListPatternMode.SHOW_LATEST_PREV): - cli_out_write(f"{indentation * 3}PID: {pref.package_id}", - fg=reference_color) - cli_out_write(f"{indentation * 4}PREV: {pref.revision}{pref_date}", - fg=field_color) - continue - for item, contents in binary_info.items(): - if not contents: - continue - cli_out_write(f"{indentation * 4}{item}:", fg=field_color) - if isinstance(contents, dict): - for k, v in contents.items(): - cli_out_write(f"{indentation * 5}{k}={v}", fg=value_color) - else: - for c in contents: - cli_out_write(f"{indentation * 5}{c}", fg=value_color) - elif search_mode in (ListPatternMode.SHOW_PACKAGE_IDS, - ListPatternMode.SHOW_ALL_PREVS, - ListPatternMode.SHOW_LATEST_PREV): - cli_out_write(f"{indentation * 3}There are no packages for this revision.", - fg=field_color) + def format_timestamps(item): + if isinstance(item, dict): + result = {} + for k, v in item.items(): + if isinstance(v, dict) and v.get("timestamp"): + timestamp = v.pop("timestamp") + k = f"{k} ({timestamp})" + result[k] = format_timestamps(v) + return result + return item + info = {remote: format_timestamps(values) for remote, values in info.items()} + print_serial(info) def print_list_json(data): results = data["results"] - myjson = json.dumps(results, indent=4, cls=ConanJSONEncoder) + myjson = json.dumps(results, indent=4) cli_out_write(myjson) @@ -113,8 +118,7 @@ def list(conan_api: ConanAPI, parser, *args): except Exception as e: results[name] = {"error": str(e)} else: - results[name] = list_bundle.serialize() if args.format in ("json", "html") \ - else list_bundle.ordered_recipes_by_name + results[name] = list_bundle.serialize() return { "results": results, "search_mode": ref_pattern.mode, diff --git a/conan/cli/commands/search.py b/conan/cli/commands/search.py index a6bb9fa9609..fc698705ba2 100644 --- a/conan/cli/commands/search.py +++ b/conan/cli/commands/search.py @@ -35,8 +35,7 @@ def search(conan_api: ConanAPI, parser, *args): except Exception as e: results[remote.name] = {"error": str(e)} else: - results[remote.name] = list_bundle.serialize() if args.format == "json" \ - else list_bundle.ordered_recipes_by_name + results[remote.name] = list_bundle.serialize() return { "results": results, "search_mode": search_mode, diff --git a/conans/test/integration/build_requires/test_toolchain_packages.py b/conans/test/integration/build_requires/test_toolchain_packages.py index 1c35cbabc3f..e923c64706c 100644 --- a/conans/test/integration/build_requires/test_toolchain_packages.py +++ b/conans/test/integration/build_requires/test_toolchain_packages.py @@ -386,4 +386,5 @@ def test(self): "build_type": "Release", "os": "Linux" } - assert settings_target == pkgs_json["Local Cache"][pref.ref.repr_notime()][pref.repr_notime()]["settings_target"] + revision_dict = pkgs_json["Local Cache"]["gcc/0.1"]["revisions"][pref.ref.revision] + assert settings_target == revision_dict["packages"][pref.package_id]["info"]["settings_target"] diff --git a/conans/test/integration/command/remove_test.py b/conans/test/integration/command/remove_test.py index 1c911973539..1d7bb74656b 100644 --- a/conans/test/integration/command/remove_test.py +++ b/conans/test/integration/command/remove_test.py @@ -164,13 +164,13 @@ def test_remove_all_packages_but_the_recipe_at_remote(self): RecipeReference.loads("foobar/0.1@user/testing")) self.client.run("list foobar/0.1@user/testing#{} -r default".format(ref.revision)) default_arch = self.client.get_default_host_profile().settings['arch'] - self.assertIn(f"arch={default_arch}", self.client.out) - self.assertIn("arch=x86", self.client.out) + self.assertIn(f"arch: {default_arch}", self.client.out) + self.assertIn("arch: x86", self.client.out) self.client.run("remove -c foobar/0.1@user/testing:* -r default") self.client.run("search foobar/0.1@user/testing -r default") - self.assertNotIn(f"arch={default_arch}", self.client.out) - self.assertNotIn("arch=x86", self.client.out) + self.assertNotIn(f"arch: {default_arch}", self.client.out) + self.assertNotIn("arch: x86", self.client.out) # populated packages of bar diff --git a/conans/test/integration/command_v2/list_test.py b/conans/test/integration/command_v2/list_test.py index 32499ba9a88..fa34e03ff3d 100644 --- a/conans/test/integration/command_v2/list_test.py +++ b/conans/test/integration/command_v2/list_test.py @@ -2,600 +2,547 @@ import os import re import textwrap +import time +from collections import OrderedDict from unittest.mock import patch, Mock import pytest from conans.errors import ConanException, ConanConnectionError -from conans.model.recipe_ref import RecipeReference -from conans.model.package_ref import PkgReference from conans.test.assets.genconanfile import GenConanfile from conans.test.utils.tools import TestClient, TestServer +from conans.util.env import environment_update -class TestListBase: - @pytest.fixture(autouse=True) - def _setup(self): - self.client = TestClient() - - def _add_remote(self, remote_name): - self.client.servers[remote_name] = TestServer(users={"username": "passwd"}, - write_permissions=[("*/*@*/*", "*")]) - self.client.update_servers() - self.client.run("remote login {} username -p passwd".format(remote_name)) - - def _upload_recipe(self, remote, ref): - self.client.save({'conanfile.py': GenConanfile()}) - ref = RecipeReference.loads(ref) - self.client.run(f"create . --name={ref.name} --version={ref.version} " - f"--user={ref.user} --channel={ref.channel}") - self.client.run("upload --force -r {} {}".format(remote, ref)) - - def _upload_full_recipe(self, remote, ref): - self.client.save({"conanfile.py": GenConanfile("pkg", "0.1").with_package_file("file.h", - "0.1")}) - self.client.run("create . --user=user --channel=channel") - self.client.run("upload --force -r {} {}".format(remote, "pkg/0.1@user/channel")) - - self.client.save({'conanfile.py': GenConanfile().with_require("pkg/0.1@user/channel") - .with_settings("os", "build_type", "arch") - .with_option("shared", [True, False]) - .with_default_option("shared", False) - }) - self.client.run(f"create . --name={ref.name} --version={ref.version} " - f"-s os=Macos -s build_type=Release -s arch=x86_64 " - f"--user={ref.user} --channel={ref.channel}") - self.client.run("upload --force -r {} {}".format(remote, ref)) - - def _upload_full_recipe_without_conaninfo(self, remote, ref): - self.client.save({"conanfile.py": GenConanfile("pkg", "0.1").with_package_file("file.h", - "0.1")}) - self.client.run("create . --user=user --channel=channel") - self.client.run("upload --force -r {} {}".format(remote, "pkg/0.1@user/channel")) - - self.client.save({'conanfile.py': GenConanfile() - }) - self.client.run(f"create . --name={ref.name} --version={ref.version} " - f"--user={ref.user} --channel={ref.channel}") - self.client.run("upload --force -r {} {}".format(remote, ref)) - - - @staticmethod - def _get_fake_recipe_refence(recipe_name): - return f"{recipe_name}#fca0383e6a43348f7989f11ab8f0a92d" - - def _get_lastest_recipe_ref(self, recipe_name): - return self.client.cache.get_latest_recipe_reference(RecipeReference.loads(recipe_name)) - - def _get_lastest_package_ref(self, pref): - return self.client.cache.get_latest_package_reference(PkgReference.loads(pref)) - - -class TestParams(TestListBase): +class TestParamErrors: def test_query_param_is_required(self): - self._add_remote("remote1") - - self.client.run("list", assert_error=True) - assert "error: the following arguments are required: reference" in self.client.out - - self.client.run("list -c", assert_error=True) - assert "error: the following arguments are required: reference" in self.client.out - - self.client.run('list -r="*"', assert_error=True) - assert "error: the following arguments are required: reference" in self.client.out - - self.client.run("list --remote remote1 --cache", assert_error=True) - assert "error: the following arguments are required: reference" in self.client.out + c = TestClient() + c.run("list", assert_error=True) + assert "error: the following arguments are required: reference" in c.out + + c.run("list -c", assert_error=True) + assert "error: the following arguments are required: reference" in c.out + + c.run('list -r="*"', assert_error=True) + assert "error: the following arguments are required: reference" in c.out + + c.run("list --remote remote1 --cache", assert_error=True) + assert "error: the following arguments are required: reference" in c.out + + +@pytest.fixture(scope="module") +def client(): + servers = OrderedDict([("default", TestServer()), + ("other", TestServer())]) + c = TestClient(servers=servers, inputs=2*["admin", "password"]) + c.save({ + "zlib.py": GenConanfile("zlib"), + "zlib_ng.py": GenConanfile("zlib_ng", "1.0.0"), + "zli.py": GenConanfile("zli", "1.0.0"), + "zli_rev2.py": GenConanfile("zli", "1.0.0").with_settings("os") + .with_package_file("f.txt", env_var="MYREV"), + "zlix.py": GenConanfile("zlix", "1.0.0"), + "test.py": GenConanfile("test", "1.0").with_requires("zlix/1.0.0") + + .with_python_requires("zlix/1.0.0"), + "conf.py": GenConanfile("conf", "1.0") + }) + c.run("create zli.py") + c.run("create zlib.py --version=1.0.0 --user=user --channel=channel") + c.run("create zlib.py --version=2.0.0 --user=user --channel=channel") + c.run("create zlix.py") + c.run("create test.py") + c.run('create conf.py -c tools.info.package_id:confs="[\'tools.build:cxxflags\']"' + ' -c tools.build:cxxflags="[\'--flag1\']"') + c.run("upload * -r=default -c") + c.run("upload * -r=other -c") + + time.sleep(1.0) + # We create and upload new revisions later, to avoid timestamp overlaps (low resolution) + with environment_update({"MYREV": "0"}): + c.run("create zli_rev2.py -s os=Windows") + c.run("create zli_rev2.py -s os=Linux") + c.run("upload * -r=default -c") + with environment_update({"MYREV": "42"}): + c.run("create zli_rev2.py -s os=Windows") + c.run("upload * -r=default -c") + return c + + +def remove_timestamps(item): + if isinstance(item, dict): + if item.get("timestamp"): + item["timestamp"] = "" + for v in item.values(): + remove_timestamps(v) + return item + + +class TestListRefs: + @staticmethod + def check(client, pattern, remote, expected): + r = "-r=default" if remote else "" + r_msg = "default" if remote else "Local Cache" + client.run(f"list {pattern} {r}") + print(client.out) + expected = textwrap.indent(expected, " ") + expected_output = f"{r_msg}\n" + expected + expected_output = re.sub(r"\(.*\)", "", expected_output) + output = re.sub(r"\(.*\)", "", str(client.out)) + assert expected_output == output -class TestRemotes(TestListBase): - def test_by_default_search_only_in_cache(self): - self._add_remote("remote1") - self._add_remote("remote2") + @staticmethod + def check_json(client, pattern, remote, expected): + r = "-r=default" if remote else "" + r_msg = "default" if remote else "Local Cache" + client.run(f"list {pattern} {r} --format=json", redirect_stdout="file.json") + list_json = client.load("file.json") + print(list_json) + list_json = json.loads(list_json) + assert remove_timestamps(list_json[r_msg]) == remove_timestamps(expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_recipes(self, client, remote): + pattern = "z*" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + zlib + zlib/1.0.0@user/channel + zlib/2.0.0@user/channel + zlix + zlix/1.0.0 + """) + self.check(client, pattern, remote, expected) + expected_json = { + "zli/1.0.0": {}, + "zlib/1.0.0@user/channel": {}, + "zlib/2.0.0@user/channel": {}, + "zlix/1.0.0": {} + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + @pytest.mark.parametrize("pattern", ["zlib", "zlib/*", "*@user/channel"]) + def test_list_recipe_versions(self, client, pattern, remote): + expected = textwrap.dedent(f"""\ + zlib + zlib/1.0.0@user/channel + zlib/2.0.0@user/channel + """) + self.check(client, pattern, remote, expected) + expected_json = { + "zlib/1.0.0@user/channel": {}, + "zlib/2.0.0@user/channel": {} + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + @pytest.mark.parametrize("pattern", ["nomatch", "nomatch*", "nomatch/*"]) + def test_list_recipe_no_match(self, client, pattern, remote): + if pattern == "nomatch": # EXACT IS AN ERROR + expected = "ERROR: Recipe 'nomatch' not found\n" + else: + expected = "WARN: There are no matching recipe references\n" + + self.check(client, pattern, remote, expected) + if pattern == "nomatch": + expected_json = {"error": "Recipe 'nomatch' not found"} + else: + expected_json = {} + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_recipe_latest_revision(self, client, remote): + # by default, when a reference is complete, we show latest recipe revision + pattern = "zli/1.0.0" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (10-11-2023 10:13:13) + """) + self.check(client, pattern, remote, expected) + expected_json = { + "zli/1.0.0": { + "revisions": { + "b58eeddfe2fd25ac3a105f72836b3360": { + "timestamp": "2023-01-10 00:25:32 UTC" + } + } + } + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_recipe_all_latest_revision(self, client, remote): + # we can show the latest revision from several matches, if we add ``#latest`` + pattern = "zlib/*#latest" + expected = textwrap.dedent(f"""\ + zlib + zlib/1.0.0@user/channel + revisions + ffd4bc45820ddb320ab224685b9ba3fb (10-11-2023 10:13:13) + zlib/2.0.0@user/channel + revisions + ffd4bc45820ddb320ab224685b9ba3fb (10-11-2023 10:13:13) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_recipe_several_revision(self, client, remote): + # we can show the latest revision from several matches, if we add ``#latest`` + pattern = "zli/1.0.0#*" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + f034dc90894493961d92dd32a9ee3b78 (10-11-2023 10:13:13) + b58eeddfe2fd25ac3a105f72836b3360 (10-11-2023 10:13:13) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_recipe_multiple_revision(self, client, remote): + pattern = "zli*#*" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + f034dc90894493961d92dd32a9ee3b78 (10-11-2023 10:13:13) + b58eeddfe2fd25ac3a105f72836b3360 (10-11-2023 10:13:13) + zlib + zlib/1.0.0@user/channel + revisions + ffd4bc45820ddb320ab224685b9ba3fb (10-11-2023 10:13:13) + zlib/2.0.0@user/channel + revisions + ffd4bc45820ddb320ab224685b9ba3fb (10-11-2023 10:13:13) + zlix + zlix/1.0.0 + revisions + 81f598d1d8648389bb7d0494fffb654e (10-11-2023 10:13:13) + """) + self.check(client, pattern, remote, expected) - expected_output = textwrap.dedent("""\ - Local Cache: - ERROR: Recipe 'whatever/0.1' not found""") - self.client.run(f"list {self._get_fake_recipe_refence('whatever/0.1')}") - assert expected_output in self.client.out +class TestListPrefs: - def test_search_no_matching_recipes(self): - self._add_remote("remote1") - self._add_remote("remote2") + @staticmethod + def check(client, pattern, remote, expected): + r = "-r=default" if remote else "" + r_msg = "default" if remote else "Local Cache" + client.run(f"list {pattern} {r}") + print(client.out) + expected = textwrap.indent(expected, " ") + expected_output = f"{r_msg}\n" + expected + expected_output = re.sub(r"\(.*\)", "", expected_output) + output = re.sub(r"\(.*\)", "", str(client.out)) + assert expected_output == output + @staticmethod + def check_json(client, pattern, remote, expected): + r = "-r=default" if remote else "" + r_msg = "default" if remote else "Local Cache" + client.run(f"list {pattern} {r} --format=json", redirect_stdout="file.json") + list_json = client.load("file.json") + print(list_json) + list_json = json.loads(list_json) + assert remove_timestamps(list_json[r_msg]) == remove_timestamps(expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_pkg_ids(self, client, remote): + pattern = "zli/1.0.0:*" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (10-11-2023 10:13:13) + packages + 9a4eb3c8701508aa9458b1a73d0633783ecc2270 + info + settings + os: Linux + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + info + settings + os: Windows + """) + self.check(client, pattern, remote, expected) + expected_json = { + "zli/1.0.0": { + "revisions": { + "b58eeddfe2fd25ac3a105f72836b3360": { + "timestamp": "2023-01-10 16:30:27 UTC", + "packages": { + "9a4eb3c8701508aa9458b1a73d0633783ecc2270": { + "info": { + "settings": { + "os": "Linux" + } + } + }, + "ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715": { + "info": { + "settings": { + "os": "Windows" + } + } + } + } + } + } + } + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_pkg_ids_confs(self, client, remote): + pattern = "conf/*:*" + expected = textwrap.dedent("""\ + conf + conf/1.0 + revisions + e4e1703f72ed07c15d73a555ec3a2fa1 (10-11-2023 10:13:13) + packages + 78c6fa29e8164ce399087ad6067c8f9e2f1c4ad0 + info + conf + tools.build:cxxflags: ['--flag1'] + """) + self.check(client, pattern, remote, expected) + expected_json = { + "conf/1.0": { + "revisions": { + "e4e1703f72ed07c15d73a555ec3a2fa1": { + "timestamp": "2023-01-10 10:07:33 UTC", + "packages": { + "78c6fa29e8164ce399087ad6067c8f9e2f1c4ad0": { + "info": { + "conf": { + "tools.build:cxxflags": "['--flag1']" + } + } + } + } + } + } + } + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_pkg_ids_requires(self, client, remote): + pattern = "test/*:*" + expected = textwrap.dedent("""\ + test + test/1.0 + revisions + 7df6048d3cb39b75618717987fb96453 (10-11-2023 10:13:13) + packages + 81d0d9a6851a0208c2bb35fdb34eb156359d939b + info + requires + zlix/1.Y.Z + python_requires + zlix/1.0.Z + """) + self.check(client, pattern, remote, expected) + expected_json = { + "test/1.0": { + "revisions": { + "7df6048d3cb39b75618717987fb96453": { + "timestamp": "2023-01-10 22:17:13 UTC", + "packages": { + "81d0d9a6851a0208c2bb35fdb34eb156359d939b": { + "info": { + "requires": [ + "zlix/1.Y.Z" + ], + "python_requires": [ + "zlix/1.0.Z" + ] + } + } + } + } + } + } + } + self.check_json(client, pattern, remote, expected_json) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_pkg_ids_all_rrevs(self, client, remote): + pattern = "zli/1.0.0#*:*" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + f034dc90894493961d92dd32a9ee3b78 (2023-01-10 22:19:58 UTC) + packages + da39a3ee5e6b4b0d3255bfef95601890afd80709 + info + b58eeddfe2fd25ac3a105f72836b3360 (2023-01-10 22:19:59 UTC) + packages + 9a4eb3c8701508aa9458b1a73d0633783ecc2270 + info + settings + os: Linux + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + info + settings + os: Windows + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_latest_prevs(self, client, remote): + pattern = "zli/1.0.0:*#latest" + # TODO: This is doing a package_id search, but not showing info + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (2023-01-10 22:27:34 UTC) + packages + 9a4eb3c8701508aa9458b1a73d0633783ecc2270 + revisions + 9beff32b8c94ea0ce5a5e67dad95f525 (10-11-2023 10:13:13) + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + revisions + d9b1e9044ee265092e81db7028ae10e0 (10-11-2023 10:13:13) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_all_prevs(self, client, remote): + pattern = "zli/1.0.0:*#*" + # TODO: This is doing a package_id search, but not showing info + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (2023-01-10 22:41:09 UTC) + packages + 9a4eb3c8701508aa9458b1a73d0633783ecc2270 + revisions + 9beff32b8c94ea0ce5a5e67dad95f525 (2023-01-10 22:41:09 UTC) + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + revisions + d9b1e9044ee265092e81db7028ae10e0 (2023-01-10 22:41:10 UTC) + 24532a030b4fcdfed699511f6bfe35d3 (2023-01-10 22:41:09 UTC) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_package_id_all_prevs(self, client, remote): + pattern = "zli/1.0.0:ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715#*" + # TODO: We might want to improve the output, grouping PREVS for the + # same package_id + expected_json = { + "zli/1.0.0": { + "revisions": { + "b58eeddfe2fd25ac3a105f72836b3360": { + "timestamp": "2023-01-10 22:45:49 UTC", + "packages": { + "ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715": { + "revisions": { + "d9b1e9044ee265092e81db7028ae10e0": { + "timestamp": "2023-01-10 22:45:49 UTC" + }, + "24532a030b4fcdfed699511f6bfe35d3": { + "timestamp": "2023-01-10 22:45:49 UTC" + } + } + } + } + } + } + } + } + self.check_json(client, pattern, remote, expected_json) + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (2023-01-10 22:41:09 UTC) + packages + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + revisions + d9b1e9044ee265092e81db7028ae10e0 (2023-01-10 22:41:10 UTC) + 24532a030b4fcdfed699511f6bfe35d3 (2023-01-10 22:41:09 UTC) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_package_id_latest_prev(self, client, remote): + pattern = "zli/1.0.0:ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715" + expected = textwrap.dedent(f"""\ + zli + zli/1.0.0 + revisions + b58eeddfe2fd25ac3a105f72836b3360 (2023-01-10 23:13:12 UTC) + packages + ebec3dc6d7f6b907b3ada0c3d3cdc83613a2b715 + revisions + d9b1e9044ee265092e81db7028ae10e0 (2023-01-10 23:13:12 UTC) + """) + self.check(client, pattern, remote, expected) + + @pytest.mark.parametrize("remote", [True, False]) + def test_list_missing_package_id(self, client, remote): + pattern = "zli/1.0.0:nonexists_id" + # TODO: The message is still different in the server + if remote: + expected = "ERROR: Binary package not found: 'zli/1.0.0@_/_#" \ + "b58eeddfe2fd25ac3a105f72836b3360:nonexists_id'. [Remote: default]\n" + else: + expected = "ERROR: Binary package not found: 'zli/1.0.0:nonexists_id\n" + self.check(client, pattern, remote, expected) + + def test_query(self): + pass + + +class TestListRemotes: + """ advanced use case: + - check multiple remotes output + """ + + def test_search_no_matching_recipes(self, client): expected_output = textwrap.dedent("""\ - Local Cache: + Local Cache ERROR: Recipe 'whatever/0.1' not found - remote1: + default ERROR: Recipe 'whatever/0.1' not found - remote2: + other ERROR: Recipe 'whatever/0.1' not found """) - rrev = self._get_fake_recipe_refence('whatever/0.1') - self.client.run(f'list -c -r="*" {rrev}') - assert expected_output == self.client.out + client.run('list -c -r="*" whatever/0.1') + assert expected_output == client.out def test_fail_if_no_configured_remotes(self): - self.client.run('list -r="*" whatever/1.0#123', assert_error=True) - assert "ERROR: Remotes for pattern '*' can't be found or are disabled" in self.client.out - - def test_search_disabled_remote(self): - self._add_remote("remote1") - self._add_remote("remote2") - self.client.run("remote disable remote1") - # He have to put both remotes instead of using "-a" because of the - # disbaled remote won't appear - self.client.run("list whatever/1.0#123 -r remote1 -r remote2", assert_error=True) - assert "ERROR: Remote 'remote1' can't be found or is disabled" in self.client.out + client = TestClient() + client.run('list -r="*" whatever/1.0#123', assert_error=True) + assert "ERROR: Remotes for pattern '*' can't be found or are disabled" in client.out @pytest.mark.parametrize("exc,output", [ (ConanConnectionError("Review your network!"), "ERROR: Review your network!"), (ConanException("Boom!"), "ERROR: Boom!") ]) - def test_search_remote_errors_but_no_raising_exceptions(self, exc, output): - self._add_remote("remote1") - self._add_remote("remote2") + def test_search_remote_errors_but_no_raising_exceptions(self, client, exc, output): with patch("conan.api.subapi.search.SearchAPI.recipes", new=Mock(side_effect=exc)): - self.client.run(f'list whatever/1.0 -r="*"') - expected_output = textwrap.dedent(f"""\ - remote1: - {output} - remote2: - {output} - """) - assert expected_output == self.client.out - - def test_search_in_missing_remote(self): - remote1 = "remote1" - - remote1_recipe1 = "test_recipe/1.0.0@user/channel" - remote1_recipe2 = "test_recipe/1.1.0@user/channel" - - expected_output = "ERROR: Remote 'wrong_remote' can't be found or is disabled" - - self._add_remote(remote1) - self._upload_recipe(remote1, remote1_recipe1) - self._upload_recipe(remote1, remote1_recipe2) - - rrev = self._get_fake_recipe_refence(remote1_recipe1) - self.client.run(f"list -r wrong_remote {rrev}", assert_error=True) - assert expected_output in self.client.out - - -class TestListUseCases(TestListBase): - - def test_list_recipes(self): - self.client.save({ - "zlib.py": GenConanfile("zlib", "1.0.0"), - "zlib2.py": GenConanfile("zlib", "2.0.0"), - "zli.py": GenConanfile("zli", "1.0.0"), - "zlix.py": GenConanfile("zlix", "1.0.0"), - }) - self.client.run("export zlib.py --user=user --channel=channel") - self.client.run("export zlib2.py --user=user --channel=channel") - self.client.run("export zli.py") - self.client.run("export zlix.py") - self.client.run(f"list z*") - expected_output = textwrap.dedent(f"""\ - Local Cache: - zlix - zlix/1.0.0 - zli - zli/1.0.0 - zlib - zlib/2.0.0@user/channel - zlib/1.0.0@user/channel - """) - assert expected_output == self.client.out - - def test_list_latest_recipe_revision_by_default(self): - self.client.save({ - "conanfile.py": GenConanfile("test_recipe", "1.0.0").with_package_file("file.h", "0.1") - }) - self.client.run("export . --user=user --channel=channel") - rrev = self._get_lastest_recipe_ref("test_recipe/1.0.0@user/channel") - self.client.run(f"list test_recipe/1.0.0@user/channel") + client.run(f'list whatever/1.0 -r="*"') expected_output = textwrap.dedent(f"""\ - Local Cache: - test_recipe - %s .* - """ % rrev.repr_notime()) - assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - - def test_list_all_the_latest_recipe_revision(self): - self.client.save({ - "hello1.py": GenConanfile("hello", "1.0.0").with_generator("CMakeToolchain"), # rrev - "hello.py": GenConanfile("hello", "1.0.0"), # latest rrev - "bye.py": GenConanfile("bye", "1.0.0") - }) - self.client.run("export hello1.py --user=user --channel=channel") - self.client.run("export hello.py --user=user --channel=channel") - hello_latest_rrev = self._get_lastest_recipe_ref("hello/1.0.0@user/channel") - self.client.run("export bye.py --user=user --channel=channel") - self.client.run(f"list *#latest") - expected_output = textwrap.dedent(f"""\ - Local Cache: - bye - bye/1.0.0@user/channel#c720a82a9c904a0450ec1aa177281ea2 .* - hello - hello/1.0.0@user/channel#7a34833afbd87d791b2201882b1afb2b .* - """) - assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - assert hello_latest_rrev.repr_notime() in expected_output - - def test_list_latest_package_revisions_by_default(self): - self.client.save({ - "conanfile.py": GenConanfile("test_recipe", "1.0.0").with_package_file("file.h", "0.1") - }) - self.client.run("create . --user=user --channel=channel") - rrev = self._get_lastest_recipe_ref("test_recipe/1.0.0@user/channel") - pid = "da39a3ee5e6b4b0d3255bfef95601890afd80709" - prev = self._get_lastest_package_ref(f"{rrev.repr_notime()}:{pid}") - self.client.run(f"list {prev.repr_notime()}") - expected_output = textwrap.dedent(f"""\ - Local Cache: - test_recipe - test_recipe/1.0.0@user/channel#ddfadce26d00a560850eb8767fe76ae4 .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 9c929aed65f04337a4143311d72fc897 .* - """) - assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - - def test_list_all_the_latest_package_revisions(self): - self.client.save({ - "hello.py": GenConanfile("hello", "1.0.0").with_package_file("file.h", "0.1"), - "bye.py": GenConanfile("bye", "1.0.0").with_package_file("file.h", "0.1") - }) - self.client.run("create hello.py --user=user --channel=channel") - self.client.run("create hello.py --user=user --channel=channel") # latest prev - self.client.run("create bye.py --user=user --channel=channel") - self.client.run("create bye.py --user=user --channel=channel") # latest prev - self.client.run("list *:*#latest") - expected_output = textwrap.dedent(f"""\ - Local Cache: - bye - bye/1.0.0@user/channel#51edd97e27e407a01be830282558c32a .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 9c929aed65f04337a4143311d72fc897 .* - hello - hello/1.0.0@user/channel#6fccfa5dd0bbb1223578c1771839eb6d .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 9c929aed65f04337a4143311d72fc897 .* - """) - assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - - def test_search_package_ids_but_empty_conan_info(self): - remote_name = "remote1" - recipe_name = "test_recipe/1.0.0@user/channel" - self._add_remote(remote_name) - self._upload_recipe(remote_name, recipe_name) - rrev = self._get_lastest_recipe_ref(recipe_name) - self.client.run(f"list {rrev.repr_notime()}:* -r remote1") - expected_output = textwrap.dedent("""\ - remote1: - test_recipe - test_recipe/1.0.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - Empty package information - """) - assert bool(re.match(expected_output, str(self.client.out), re.MULTILINE)) - - def test_search_package_ids_from_latest_rrev_in_all_remotes_and_cache(self): - remote1 = "remote1" - remote2 = "remote2" - - self._add_remote(remote1) - self._upload_full_recipe(remote1, RecipeReference(name="test_recipe", version="1.0", - user="user", channel="channel")) - self._add_remote(remote2) - self._upload_full_recipe(remote2, RecipeReference(name="test_recipe", version="2.1", - user="user", channel="channel")) - self.client.run(f'list test_recipe/*:* -r="*" -c') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - Local Cache: - test_recipe - test_recipe/2.1@user/channel#a22316c3831b70763e4405841ee93f27 .* - PID: 630ddee056279fad89b691ac0f36eb084f40da38 .* - settings: - arch=x86_64 - build_type=Release - os=Macos - options: - shared=False - requires: - pkg/0.1.Z@user/channel - test_recipe/1.0@user/channel#a22316c3831b70763e4405841ee93f27 .* - PID: 630ddee056279fad89b691ac0f36eb084f40da38 .* - settings: - arch=x86_64 - build_type=Release - os=Macos - options: - shared=False - requires: - pkg/0.1.Z@user/channel - remote1: - test_recipe - test_recipe/1.0@user/channel#a22316c3831b70763e4405841ee93f27 .* - PID: 630ddee056279fad89b691ac0f36eb084f40da38 - settings: - arch=x86_64 - build_type=Release - os=Macos - options: - shared=False - requires: - pkg/0.1.Z@user/channel - remote2: - test_recipe - test_recipe/2.1@user/channel#a22316c3831b70763e4405841ee93f27 .* - PID: 630ddee056279fad89b691ac0f36eb084f40da38 - settings: - arch=x86_64 - build_type=Release - os=Macos - options: - shared=False - requires: - pkg/0.1.Z@user/channel - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - - def test_search_all_revisions_and_package_revisions(self): - """Checking if RREVs and PREVs are shown correctly""" - remote1 = "remote1" - remote2 = "remote2" - - self._add_remote(remote1) - self._upload_full_recipe_without_conaninfo(remote1, - RecipeReference(name="test_recipe", version="1.0", - user="user", channel="channel")) - self._add_remote(remote2) - self._upload_full_recipe_without_conaninfo(remote2, - RecipeReference(name="test_recipe", version="2.1", - user="user", channel="channel")) - self.client.run(f'list *#* -r="*" -c') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - Local Cache: - test_recipe - test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - pkg - pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* - remote1: - pkg - pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* - test_recipe - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - remote2: - pkg - pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* - test_recipe - test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - self.client.run(f'list test_recipe/*:*#* -r="*" -c') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - Local Cache: - test_recipe - test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* - remote1: - test_recipe - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* - remote2: - test_recipe - test_recipe/2.1@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - - def test_search_all_revisions_given_a_package_id(self): - """ - Checking if PREVs are shown correctly given a PkgID and even though that package has no - configuration at all. - """ - remote1 = "remote1" - self._add_remote(remote1) - self._upload_full_recipe_without_conaninfo(remote1, - RecipeReference(name="test_recipe", version="1.0", - user="user", channel="channel")) - self.client.run(f'list *:* -r=remote1') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - remote1: - pkg - pkg/0.1@user/channel#44a36b797bc85fb66af6acf90cf8f539 .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - Empty package information - test_recipe - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - Empty package information - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - self.client.run(f'list test_recipe/*:da39a3ee5e6b4b0d3255bfef95601890afd80709#* -r=remote1') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - remote1: - test_recipe - test_recipe/1.0@user/channel#4d670581ccb765839f2239cc8dff8fbd .* - PID: da39a3ee5e6b4b0d3255bfef95601890afd80709 - PREV: 0ba8627bd47edc3a501e8f0eb9a79e5e .* - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - - def test_list_package_query_options(self): - self.client.save({"conanfile.py": GenConanfile("pkg", "0.1") - .with_package_file("file.h", "0.1") - .with_settings("os", "build_type", "arch")}) - self.client.run("create . --user=user --channel=channel " - "-s os=Windows -s build_type=Release -s arch=x86_64") - self.client.run("create . --user=user --channel=channel " - "-s os=Macos -s build_type=Release -s arch=x86_64") - self.client.run("create . --user=user --channel=channel " - "-s os=Macos -s build_type=Release -s arch=armv7") - self.client.run(f'list pkg/0.1#*:*') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - Local Cache: - pkg - pkg/0.1@user/channel#89ab3ffd306cb65a8ca8e2a1c8b96aae .* - PID: 5f2a74726e897f644b3f42dea59faecf8eee2b50 .* - settings: - arch=armv7 - build_type=Release - os=Macos - PID: 723257509aee8a72faf021920c2874abc738e029 .* - settings: - arch=x86_64 - build_type=Release - os=Windows - PID: 9ac8640923e5284645f8852ef8ba335654f4020e .* - settings: - arch=x86_64 - build_type=Release - os=Macos - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - self.client.run(f'list pkg/0.1#*:* -p os=Windows') - output = str(self.client.out) - expected_output = textwrap.dedent("""\ - Local Cache: - pkg - pkg/0.1@user/channel#89ab3ffd306cb65a8ca8e2a1c8b96aae .* - PID: 723257509aee8a72faf021920c2874abc738e029 .* - settings: - arch=x86_64 - build_type=Release - os=Windows - """) - assert bool(re.match(expected_output, output, re.MULTILINE)) - - -class TestListPackages: - def test_list_package_info_and_json_format(self): - c = TestClient(default_server_user=True) - c.save({"dep/conanfile.py": GenConanfile("dep", "1.2.3"), - "pkg/conanfile.py": GenConanfile("pkg", "2.3.4").with_requires("dep/1.2.3") - .with_settings("os", "arch").with_shared_option(False)}) - c.run("create dep") - c.run("create pkg -s os=Windows -s arch=x86") - c.run("list pkg/2.3.4#0fc07368b81b38197adc73ee2cb89da8") - expected_output = textwrap.dedent(f"""\ - Local Cache: - pkg - pkg/2.3.4#0fc07368b81b38197adc73ee2cb89da8 .* - PID: ec080285423a5e38126f0d5d51b524cf516ff7a5 .* - settings: - arch=x86 - os=Windows - options: - shared=False - requires: - dep/1.2.Z - """) - assert bool(re.match(expected_output, c.out, re.MULTILINE)) - - rrev = "pkg/2.3.4#0fc07368b81b38197adc73ee2cb89da8" - c.run(f"list {rrev} --format=json", redirect_stdout="packages.json") - pkgs_json = c.load("packages.json") - pkgs_json = json.loads(pkgs_json) - pref = "pkg/2.3.4#0fc07368b81b38197adc73ee2cb89da8:ec080285423a5e38126f0d5d51b524cf516ff7a5" - assert pkgs_json["Local Cache"][rrev][pref]["settings"]["os"] == "Windows" - - def test_list_packages_with_conf(self): - """Test that tools.info.package_id:confs works, affecting the package_id and - can be listed when we are listing packages - """ - client = TestClient() - conanfile = GenConanfile().with_settings("os") - profile = textwrap.dedent(f""" - [conf] - tools.info.package_id:confs=["tools.build:cxxflags", "tools.build:cflags"] - tools.build:cxxflags=["--flag1", "--flag2"] - tools.build:cflags+=["--flag3", "--flag4"] - tools.build:sharedlinkflags=+["--flag5", "--flag6"] + default + {output} + other + {output} """) - client.save({"conanfile.py": conanfile, "profile": profile}) - client.run('create . --name=pkg --version=0.1 -s os=Windows -pr profile') - client.assert_listed_binary({"pkg/0.1": ("89d32f25195a77f4ae2e77414b870781853bdbc1", - "Build")}) - revision = client.exported_recipe_revision() - client.run(f"list pkg/0.1#{revision}") - expected_output = textwrap.dedent("""\ - Local Cache: - pkg - pkg/0.1#db6569e42e3e9916209e2ef64d6a7b52 .* - PID: 89d32f25195a77f4ae2e77414b870781853bdbc1 .* - settings: - os=Windows - conf: - tools.build:cflags=\['--flag3', '--flag4'\] - tools.build:cxxflags=\['--flag1', '--flag2'\] - """) - assert bool(re.match(expected_output, client.out, re.MULTILINE)) - - def test_list_packages_python_requires(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("export . --name=tool --version=1.1.1") - conanfile = textwrap.dedent(""" - from conan import ConanFile - class Pkg(ConanFile): - python_requires ="tool/[*]" - """) - client.save({"conanfile.py": conanfile}) - client.run("create . --name foo --version 1.0") - client.run('list foo/1.0:*') - - expected_output = textwrap.dedent("""\ - Local Cache: - foo - foo/1.0#b2ab5ffa95e8c5c19a5d1198be33103a .* - PID: 170e82ef3a6bf0bbcda5033467ab9d7805b11d0b .* - python_requires: - tool/1.1.Z - """) - assert bool(re.match(expected_output, client.out, re.MULTILINE)) - - def test_list_packages_build_requires(self): - client = TestClient() - client.save({"conanfile.py": GenConanfile()}) - client.run("create . --name=tool --version=1.1.1") - conanfile = textwrap.dedent(""" - from conan import ConanFile - class Pkg(ConanFile): - def requirements(self): - # We set the package_id_mode so it is part of the package_id - self.tool_requires("tool/1.1.1", package_id_mode="minor_mode") - """) - client.save({"conanfile.py": conanfile}) - client.run("create . --name foo --version 1.0") - client.run('list foo/1.0:*') - - expected_output = textwrap.dedent("""\ - Local Cache: - foo - foo/1.0#75821be6dc510628d538fffb2f00a51f .* - PID: d01be73a295dca843e5e198334f86ae7038423d7 .* - build_requires: - tool/1.1.Z - """) - assert bool(re.match(expected_output, client.out, re.MULTILINE)) + assert expected_output == client.out class TestListHTML: diff --git a/conans/test/integration/command_v2/search_test.py b/conans/test/integration/command_v2/search_test.py index 2ca1d113e1b..9a906c9cf03 100644 --- a/conans/test/integration/command_v2/search_test.py +++ b/conans/test/integration/command_v2/search_test.py @@ -28,9 +28,9 @@ def test_search_no_params(self): assert "error: the following arguments are required: reference" in self.client.out def test_search_no_matching_recipes(self, remotes): - expected_output = ("remote1:\n" + expected_output = ("remote1\n" " ERROR: Recipe 'whatever' not found\n" - "remote2:\n" + "remote2\n" " ERROR: Recipe 'whatever' not found\n") self.client.run("search whatever") @@ -86,9 +86,9 @@ def test_search_remote_errors_but_no_raising_exceptions(self, exc, output): with patch("conan.api.subapi.search.SearchAPI.recipes", new=Mock(side_effect=exc)): self.client.run("search whatever") expected_output = textwrap.dedent(f"""\ - remote1: + remote1 {output} - remote2: + remote2 {output} """) assert expected_output == self.client.out @@ -107,7 +107,7 @@ def test_search_by_name(self): self.client.run("search -r {} {}".format(remote_name, "test_recipe")) expected_output = ( - "remote1:\n" + "remote1\n" " test_recipe\n" " {}\n".format(recipe_name) ) @@ -124,11 +124,11 @@ def test_search_in_all_remotes(self): remote2_recipe2 = "test_recipe/2.1.0@user/channel" expected_output = ( - "remote1:\n" + "remote1\n" " test_recipe\n" " test_recipe/1.0.0@user/channel\n" " test_recipe/1.1.0@user/channel\n" - "remote2:\n" + "remote2\n" " test_recipe\n" " test_recipe/2.0.0@user/channel\n" " test_recipe/2.1.0@user/channel\n" @@ -155,7 +155,7 @@ def test_search_in_one_remote(self): remote2_recipe2 = "test_recipe/2.1.0@user/channel" expected_output = ( - "remote1:\n" + "remote1\n" " test_recipe\n" " test_recipe/1.0.0@user/channel\n" " test_recipe/1.1.0@user/channel\n" @@ -183,11 +183,11 @@ def test_search_package_found_in_one_remote(self): remote2_recipe2 = "another_recipe/2.1.0@user/channel" expected_output = ( - "remote1:\n" + "remote1\n" " test_recipe\n" " test_recipe/1.0.0@user/channel\n" " test_recipe/1.1.0@user/channel\n" - "remote2:\n" + "remote2\n" " ERROR: Recipe 'test_recipe' not found\n" ) @@ -225,7 +225,7 @@ def test_search_wildcard(self): remote1_recipe4 = "test_another/4.1.0@user/channel" expected_output = ( - "remote1:\n" + "remote1\n" " test_another\n" " test_another/2.1.0@user/channel\n" " test_another/4.1.0@user/channel\n" diff --git a/conans/test/integration/remote/test_conaninfo_parsing.py b/conans/test/integration/remote/test_conaninfo_parsing.py index 718d306e87b..ad146a1cb82 100644 --- a/conans/test/integration/remote/test_conaninfo_parsing.py +++ b/conans/test/integration/remote/test_conaninfo_parsing.py @@ -25,9 +25,9 @@ class Recipe(ConanFile): t.save({"conanfile.py": conanfile}) t.run("create . ") t.run('list weird_info/1.0:*') - assert 'ññ¨¨&是=][{}"是是是' in t.out + assert 'ññ¨¨&是: ][{}"是是是' in t.out t.run("upload * -c -r default") # TODO: I have struggled with this, it was not accepting "latest", revision needs explicit one t.run('list weird_info/1.0#8c9e59246220eef8ca3bd4ac4f39ceb3 -r default') - assert 'ññ¨¨&是=][{}"是是是' in t.out + assert 'ññ¨¨&是: ][{}"是是是' in t.out