Merge branch 'release/0.19.1'

conans/__init__.py

@@ -13,4 +13,4 @@
 COMPLEX_SEARCH_CAPABILITY = "complex_search"
 SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ]
 
-__version__ = '0.19.0'
+__version__ = '0.19.1'

conans/client/manager.py

@@ -580,6 +580,11 @@ def copy(self, reference, package_ids, username, channel, force=False):
         @param channel: Destination channel
         @param remote: install only from that remote
         """
+        # It is necessary to make sure the sources are complete before proceeding
+        remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, None)
+        remote_proxy.complete_recipe_sources(reference)
+
+        # Now we can actually copy
         output = ScopedOutput(str(reference), self._user_io.out)
         conan_file_path = self._client_cache.conanfile(reference)
         conanfile = self._loader().load_conan(conan_file_path, output)

conans/client/proxy.py

@@ -238,11 +238,7 @@ def _retrieve_from_remote(remote):
 
         raise ConanException("No remote defined")
 
-    def upload_conan(self, conan_reference, retry, retry_wait):
-        """ upload to defined remote in (-r=remote), to current remote
-        or to default remote, in that order.
-        If the remote is not set, set it
-        """
+    def complete_recipe_sources(self, conan_reference, force_complete=True):
         export_path = self._client_cache.export(conan_reference)
         sources_folder = os.path.join(export_path, EXPORT_SOURCES_DIR)
         ignore_deleted_file = None
@@ -253,14 +249,22 @@ def upload_conan(self, conan_reference, retry, retry_wait):
             if not current_remote:
                 raise ConanException("Trying to upload a package recipe without sources, "
                                      "and the remote for the sources no longer exists")
-            if current_remote != upload_remote:
+            if force_complete or current_remote != upload_remote:
                 # If uploading to a different remote than the one from which the recipe
                 # was retrieved, we definitely need to get the sources, so the recipe is complete
                 self.get_recipe_sources(conan_reference)
             else:
                 # But if same remote, no need to upload again the TGZ, it is already in the server
                 # But the upload API needs to know it to not remove the server file.
                 ignore_deleted_file = EXPORT_SOURCES_TGZ_NAME
+        return ignore_deleted_file
+
+    def upload_conan(self, conan_reference, retry, retry_wait):
+        """ upload to defined remote in (-r=remote), to current remote
+        or to default remote, in that order.
+        If the remote is not set, set it
+        """
+        ignore_deleted_file = self.complete_recipe_sources(conan_reference, force_complete=False)
 
         remote, ref_remote = self._get_remote(conan_reference)
 

conans/test/integration/export_sources_test.py

@@ -20,6 +20,19 @@ def package(self):
         self.copy("*.h", "include")
 """
 
+combined_conanfile = """
+from conans import ConanFile
+
+class HelloConan(ConanFile):
+    name = "Hello"
+    version = "0.1"
+    exports_sources = "*.h", "*.cpp"
+    exports = "*.txt"
+    def package(self):
+        self.copy("*.h", "include")
+        self.copy("data.txt", "docs")
+"""
+
 
 class ExportsSourcesTest(unittest.TestCase):
 
@@ -37,19 +50,24 @@ def setUp(self):
         self.source_folder = self.client.client_cache.source(self.reference)
         self.package_folder = self.client.client_cache.package(self.package_reference)
         self.export_folder = self.client.client_cache.export(self.reference)
-        self.export_sources_folder = os.path.join(self.export_folder, EXPORT_SOURCES_DIR)
 
-    def _check_source_folder(self):
+    def _check_source_folder(self, mode):
         """ Source folder MUST be always the same
         """
-        expected_sources = sorted(['conanfile.py', 'conanmanifest.txt', "hello.h"])
+        expected_sources = ['conanfile.py', 'conanmanifest.txt', "hello.h"]
+        if mode == "both":
+            expected_sources.append("data.txt")
+        expected_sources = sorted(expected_sources)
         self.assertEqual(sorted(os.listdir(self.source_folder)), expected_sources)
 
-    def _check_package_folder(self):
+    def _check_package_folder(self, mode):
         """ Package folder must be always the same (might have tgz after upload)
         """
-        expected_package = sorted(["conaninfo.txt", "conanmanifest.txt",
-                                   os.sep.join(["include", "hello.h"])])
+        expected_package = ["conaninfo.txt", "conanmanifest.txt",
+                            os.sep.join(["include", "hello.h"])]
+        if mode == "both":
+            expected_package.append(os.sep.join(["docs", "data.txt"]))
+        expected_package = sorted(expected_package)
         self.assertEqual(sorted(relative_dirs(self.package_folder)), expected_package)
 
     def _check_server_folder(self, mode, server=None):
@@ -58,22 +76,32 @@ def _check_server_folder(self, mode, server=None):
                                       'conanmanifest.txt'])
         if mode == "exports":
             expected_server = sorted([EXPORT_TGZ_NAME, 'conanfile.py', 'conanmanifest.txt'])
+        if mode == "both":
+            expected_server = sorted([EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, 'conanfile.py',
+                                      'conanmanifest.txt'])
         server = server or self.server
         self.assertEqual(sorted(os.listdir(server.paths.export(self.reference))),
                          expected_server)
 
-    def _check_export_folder(self, mode):
+    def _check_export_folder(self, mode, export_folder=None):
         if mode == "exports_sources":
             expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt'])
             expected_exports_sources = ["hello.h"]
         if mode == "exports":
             expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt',
                                        "hello.h"])
             expected_exports_sources = []
+        if mode == "both":
+            expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt',
+                                       "data.txt"])
+            expected_exports_sources = ["hello.h"]
+
+        export_folder = export_folder or self.export_folder
+        export_sources_folder = os.path.join(export_folder, EXPORT_SOURCES_DIR)
         # The export folder might contain or not temporary Python files
         cached = "__pycache__" if six.PY3 else "conanfile.pyc"
-        exports = [f for f in os.listdir(self.export_folder) if f != cached]
-        exports_sources = [f for f in os.listdir(self.export_sources_folder) if f != cached]
+        exports = [f for f in os.listdir(export_folder) if f != cached]
+        exports_sources = [f for f in os.listdir(export_sources_folder) if f != cached]
         self.assertEqual(sorted(exports), expected_exports)
         self.assertEqual(sorted(exports_sources), expected_exports_sources)
 
@@ -82,32 +110,43 @@ def _check_export_installed_folder(self, mode, reuploaded=False, updated=False):
         """
         if mode == "exports_sources":
             expected_exports = ['conanfile.py', 'conanmanifest.txt']
-
+        if mode == "both":
+            expected_exports = ['conanfile.py', 'conanmanifest.txt', "data.txt"]
+            if reuploaded:
+                expected_exports.append("conan_export.tgz")
         if mode == "exports":
             expected_exports = ['conanfile.py', 'conanmanifest.txt', "hello.h"]
             if reuploaded:
                 expected_exports.append("conan_export.tgz")
         if updated:
             expected_exports.append("license.txt")
         expected_exports = sorted(expected_exports)
+        export_sources_folder = os.path.join(self.export_folder, EXPORT_SOURCES_DIR)
         # The export folder might contain or not temporary Python files
         cached = "__pycache__" if six.PY3 else "conanfile.pyc"
         exports = [f for f in os.listdir(self.export_folder) if f != cached]
         self.assertEqual(sorted(exports), expected_exports)
-        self.assertFalse(os.path.exists(self.export_sources_folder))
+        self.assertFalse(os.path.exists(export_sources_folder))
 
-    def _check_export_uploaded_folder(self, mode):
+    def _check_export_uploaded_folder(self, mode, export_folder=None):
         if mode == "exports_sources":
             expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
                                            'conanmanifest.txt', EXPORT_SOURCES_TGZ_NAME])
             expected_exports_sources = ["hello.h"]
+        if mode == "both":
+            expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
+                                           'conanmanifest.txt', "data.txt", EXPORT_TGZ_NAME,
+                                           EXPORT_SOURCES_TGZ_NAME])
+            expected_exports_sources = ["hello.h"]
         if mode == "exports":
             expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
                                            'conanmanifest.txt', "hello.h", EXPORT_TGZ_NAME])
             expected_exports_sources = []
+        export_folder = export_folder or self.export_folder
+        export_sources_folder = os.path.join(export_folder, EXPORT_SOURCES_DIR)
         cached = "__pycache__" if six.PY3 else "conanfile.pyc"
-        exports = [f for f in os.listdir(self.export_folder) if f != cached]
-        exports_sources = [f for f in os.listdir(self.export_sources_folder) if f != cached]
+        exports = [f for f in os.listdir(export_folder) if f != cached]
+        exports_sources = [f for f in os.listdir(export_sources_folder) if f != cached]
         self.assertEqual(sorted(exports), expected_pkg_exports)
         self.assertEqual(sorted(exports_sources), expected_exports_sources)
 
@@ -118,16 +157,50 @@ def _check_manifest(self, mode):
         if mode == "exports_sources":
             self.assertIn("%s/hello.h: 5d41402abc4b2a76b9719d911017c592" % EXPORT_SOURCES_DIR,
                           manifest.splitlines())
-        else:
+        elif mode == "exports":
             self.assertIn("hello.h: 5d41402abc4b2a76b9719d911017c592",
                           manifest.splitlines())
+        else:
+            self.assertIn("data.txt: 8d777f385d3dfec8815d20f7496026dc", manifest.splitlines())
+            self.assertIn("%s/hello.h: 5d41402abc4b2a76b9719d911017c592" % EXPORT_SOURCES_DIR,
+                          manifest.splitlines())
 
-    @parameterized.expand([("exports", ), ("exports_sources", )])
-    def export_test(self, mode):
+    def _create_code(self, mode):
         conanfile = conanfile_py if mode == "exports" else conanfile_py.replace("exports",
                                                                                 "exports_sources")
+        if mode == "both":
+            conanfile = combined_conanfile
         self.client.save({"conanfile.py": conanfile,
-                          "hello.h": "hello"})
+                          "hello.h": "hello",
+                          "data.txt": "data"})
+
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    def copy_test(self, mode):
+        # https://github.com/conan-io/conan/issues/943
+        self._create_code(mode)
+
+        self.client.run("export lasote/testing")
+        self.client.run("install Hello/0.1@lasote/testing --build=missing")
+        self.client.run("upload Hello/0.1@lasote/testing --all")
+        self.client.run('remove Hello/0.1@lasote/testing -f')
+        self.client.run("install Hello/0.1@lasote/testing")
+
+        # new copied package data
+        reference = ConanFileReference.loads("Hello/0.1@lasote/stable")
+        source_folder = self.client.client_cache.source(reference)
+        export_folder = self.client.client_cache.export(reference)
+
+        self.client.run("copy Hello/0.1@lasote/testing lasote/stable")
+        self._check_export_folder(mode, export_folder)
+
+        self.client.run("upload Hello/0.1@lasote/stable")
+        self.assertFalse(os.path.exists(source_folder))
+        self._check_export_uploaded_folder(mode, export_folder)
+        self._check_server_folder(mode)
+
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    def export_test(self, mode):
+        self._create_code(mode)
 
         self.client.run("export lasote/testing")
         self._check_export_folder(mode)
@@ -136,8 +209,8 @@ def export_test(self, mode):
         self.client.run("install Hello/0.1@lasote/testing --build=missing")
         # Source folder and package should be exatly the same
         self._check_export_folder(mode)
-        self._check_source_folder()
-        self._check_package_folder()
+        self._check_source_folder(mode)
+        self._check_package_folder(mode)
 
         # upload to remote
         self.client.run("upload Hello/0.1@lasote/testing --all")
@@ -152,14 +225,14 @@ def export_test(self, mode):
         self.client.run("install Hello/0.1@lasote/testing")
         self.assertFalse(os.path.exists(self.source_folder))
         self._check_export_installed_folder(mode)
-        self._check_package_folder()
+        self._check_package_folder(mode)
 
         # Manifests must work too!
         self.client.run("install Hello/0.1@lasote/testing --manifests")
         self.assertFalse(os.path.exists(self.source_folder))
         # The manifests retrieve the normal state, as it retrieves sources
         self._check_export_folder(mode)
-        self._check_package_folder()
+        self._check_package_folder(mode)
         self._check_manifest(mode)
 
         # lets try to verify
@@ -169,15 +242,12 @@ def export_test(self, mode):
         self.assertFalse(os.path.exists(self.source_folder))
         # The manifests retrieve the normal state, as it retrieves sources
         self._check_export_folder(mode)
-        self._check_package_folder()
+        self._check_package_folder(mode)
         self._check_manifest(mode)
 
-    @parameterized.expand([("exports", ), ("exports_sources", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
     def export_upload_test(self, mode):
-        conanfile = conanfile_py if mode == "exports" else conanfile_py.replace("exports",
-                                                                                "exports_sources")
-        self.client.save({"conanfile.py": conanfile,
-                          "hello.h": "hello"})
+        self._create_code(mode)
 
         self.client.run("export lasote/testing")
 
@@ -193,24 +263,21 @@ def export_upload_test(self, mode):
         # install from remote
         self.client.run("install Hello/0.1@lasote/testing --build")
         self._check_export_folder(mode)
-        self._check_source_folder()
-        self._check_package_folder()
+        self._check_source_folder(mode)
+        self._check_package_folder(mode)
 
         # Manifests must work too!
         self.client.run("install Hello/0.1@lasote/testing --manifests")
         # The manifests retrieve the normal state, as it retrieves sources
         self._check_export_folder(mode)
-        self._check_package_folder()
+        self._check_package_folder(mode)
         self._check_manifest(mode)
 
-    @parameterized.expand([("exports", ), ("exports_sources", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
     def reupload_test(self, mode):
         """ try to reupload to same and other remote
         """
-        conanfile = conanfile_py if mode == "exports" else conanfile_py.replace("exports",
-                                                                                "exports_sources")
-        self.client.save({"conanfile.py": conanfile,
-                          "hello.h": "hello"})
+        self._create_code(mode)
 
         self.client.run("export lasote/testing")
         self.client.run("install Hello/0.1@lasote/testing --build=missing")
@@ -227,12 +294,9 @@ def reupload_test(self, mode):
         self._check_export_uploaded_folder(mode)
         self._check_server_folder(mode, self.other_server)
 
-    @parameterized.expand([("exports", ), ("exports_sources", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
     def update_test(self, mode):
-        conanfile = conanfile_py if mode == "exports" else conanfile_py.replace("exports",
-                                                                                "exports_sources")
-        self.client.save({"conanfile.py": conanfile,
-                          "hello.h": "hello"})
+        self._create_code(mode)
 
         self.client.run("export lasote/testing")
         self.client.run("install Hello/0.1@lasote/testing --build=missing")