fixed issues with nested exports_sources (#988)

conans/client/source.py

@@ -7,6 +7,17 @@
 import shutil
 
 
+def _merge_directories(src, dst):
+    for src_dir, _, files in os.walk(src):
+        dst_dir = os.path.join(dst, os.path.relpath(src_dir, src))
+        if not os.path.exists(dst_dir):
+            os.makedirs(dst_dir)
+        for file_ in files:
+            src_file = os.path.join(src_dir, file_)
+            dst_file = os.path.join(dst_dir, file_)
+            shutil.move(src_file, dst_file)
+
+
 def config_source(export_folder, src_folder, conan_file, output, force=False):
     """ creates src folder and retrieve, calling source() from conanfile
     the necessary source code
@@ -43,11 +54,9 @@ def remove_source(raise_error=True):
         # Now move the export-sources to the right location
         source_sources_folder = os.path.join(src_folder, EXPORT_SOURCES_DIR)
         if os.path.exists(source_sources_folder):
-            for filename in os.listdir(source_sources_folder):
-                shutil.move(os.path.join(source_sources_folder, filename),
-                            os.path.join(src_folder, filename))
+            _merge_directories(source_sources_folder, src_folder)
             # finally remove copied folder
-            os.rmdir(source_sources_folder)
+            shutil.rmtree(source_sources_folder)
         for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", CONANFILE+"o"):
             try:
                 os.remove(os.path.join(src_folder, f))

conans/test/integration/export_sources_test.py

@@ -4,10 +4,11 @@
 import os
 from conans.paths import EXPORT_SOURCES_DIR, EXPORT_SOURCES_TGZ_NAME, EXPORT_TGZ_NAME
 from nose_parameterized.parameterized import parameterized
-from conans.util.files import relative_dirs, load, save, md5sum
-import six
+from conans.util.files import load, save, md5sum
 from conans.model.manifest import FileTreeManifest
 from collections import OrderedDict
+from conans.test.utils.test_files import scan_folder
+
 
 conanfile_py = """
 from conans import ConanFile
@@ -20,6 +21,7 @@ def package(self):
         self.copy("*.h", "include")
 """
 
+
 combined_conanfile = """
 from conans import ConanFile
 
@@ -34,6 +36,34 @@ def package(self):
 """
 
 
+nested_conanfile = """
+from conans import ConanFile
+
+class HelloConan(ConanFile):
+    name = "Hello"
+    version = "0.1"
+    exports_sources = "src/*.h", "src/*.cpp"
+    exports = "src/*.txt"
+    def package(self):
+        self.copy("*.h", "include")
+        self.copy("*data.txt", "docs")
+"""
+
+
+overlap_conanfile = """
+from conans import ConanFile
+
+class HelloConan(ConanFile):
+    name = "Hello"
+    version = "0.1"
+    exports_sources = "src/*.h", "*.txt"
+    exports = "src/*.txt", "*.h"
+    def package(self):
+        self.copy("*.h", "include")
+        self.copy("*data.txt", "docs")
+"""
+
+
 class ExportsSourcesTest(unittest.TestCase):
 
     def setUp(self):
@@ -57,53 +87,57 @@ def _check_source_folder(self, mode):
         expected_sources = ['conanfile.py', 'conanmanifest.txt', "hello.h"]
         if mode == "both":
             expected_sources.append("data.txt")
+        if mode == "nested" or mode == "overlap":
+            expected_sources = ['conanfile.py', 'conanmanifest.txt', "src/hello.h", "src/data.txt"]
         expected_sources = sorted(expected_sources)
-        self.assertEqual(sorted(os.listdir(self.source_folder)), expected_sources)
+        self.assertEqual(scan_folder(self.source_folder), expected_sources)
 
     def _check_package_folder(self, mode):
         """ Package folder must be always the same (might have tgz after upload)
         """
-        expected_package = ["conaninfo.txt", "conanmanifest.txt",
-                            os.sep.join(["include", "hello.h"])]
+        if mode in ["exports", "exports_sources"]:
+            expected_package = ["conaninfo.txt", "conanmanifest.txt", "include/hello.h"]
         if mode == "both":
-            expected_package.append(os.sep.join(["docs", "data.txt"]))
-        expected_package = sorted(expected_package)
-        self.assertEqual(sorted(relative_dirs(self.package_folder)), expected_package)
+            expected_package = ["conaninfo.txt", "conanmanifest.txt", "include/hello.h",
+                                "docs/data.txt"]
+        if mode == "nested" or mode == "overlap":
+            expected_package = ["conaninfo.txt", "conanmanifest.txt", "include/src/hello.h",
+                                "docs/src/data.txt"]
+
+        self.assertEqual(scan_folder(self.package_folder), sorted(expected_package))
 
     def _check_server_folder(self, mode, server=None):
         if mode == "exports_sources":
-            expected_server = sorted([EXPORT_SOURCES_TGZ_NAME, 'conanfile.py',
-                                      'conanmanifest.txt'])
+            expected_server = [EXPORT_SOURCES_TGZ_NAME, 'conanfile.py', 'conanmanifest.txt']
         if mode == "exports":
-            expected_server = sorted([EXPORT_TGZ_NAME, 'conanfile.py', 'conanmanifest.txt'])
-        if mode == "both":
-            expected_server = sorted([EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, 'conanfile.py',
-                                      'conanmanifest.txt'])
+            expected_server = [EXPORT_TGZ_NAME, 'conanfile.py', 'conanmanifest.txt']
+        if mode == "both" or mode == "nested" or mode == "overlap":
+            expected_server = [EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, 'conanfile.py',
+                               'conanmanifest.txt']
+
         server = server or self.server
-        self.assertEqual(sorted(os.listdir(server.paths.export(self.reference))),
-                         expected_server)
+        self.assertEqual(scan_folder(server.paths.export(self.reference)), expected_server)
 
     def _check_export_folder(self, mode, export_folder=None):
         if mode == "exports_sources":
-            expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt'])
-            expected_exports_sources = ["hello.h"]
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "hello.h"),
+                                'conanfile.py', 'conanmanifest.txt']
         if mode == "exports":
-            expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt',
-                                       "hello.h"])
-            expected_exports_sources = []
+            expected_exports = ["hello.h", 'conanfile.py', 'conanmanifest.txt']
         if mode == "both":
-            expected_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py', 'conanmanifest.txt',
-                                       "data.txt"])
-            expected_exports_sources = ["hello.h"]
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "hello.h"),
+                                'conanfile.py', 'conanmanifest.txt', "data.txt"]
+        if mode == "nested":
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "src/hello.h"),
+                                "src/data.txt", 'conanfile.py', 'conanmanifest.txt']
+        if mode == "overlap":
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "src/hello.h"),
+                                "%s/%s" % (EXPORT_SOURCES_DIR, "src/data.txt"),
+                                "src/data.txt", "src/hello.h", 'conanfile.py', 'conanmanifest.txt']
 
         export_folder = export_folder or self.export_folder
-        export_sources_folder = os.path.join(export_folder, EXPORT_SOURCES_DIR)
-        # The export folder might contain or not temporary Python files
-        cached = "__pycache__" if six.PY3 else "conanfile.pyc"
-        exports = [f for f in os.listdir(export_folder) if f != cached]
-        exports_sources = [f for f in os.listdir(export_sources_folder) if f != cached]
-        self.assertEqual(sorted(exports), expected_exports)
-        self.assertEqual(sorted(exports_sources), expected_exports_sources)
+        self.assertTrue(os.path.exists(os.path.join(export_folder, EXPORT_SOURCES_DIR)))
+        self.assertEqual(scan_folder(export_folder), sorted(expected_exports))
 
     def _check_export_installed_folder(self, mode, reuploaded=False, updated=False):
         """ Just installed, no EXPORT_SOURCES_DIR is present
@@ -118,37 +152,44 @@ def _check_export_installed_folder(self, mode, reuploaded=False, updated=False):
             expected_exports = ['conanfile.py', 'conanmanifest.txt', "hello.h"]
             if reuploaded:
                 expected_exports.append("conan_export.tgz")
+        if mode == "nested":
+            expected_exports = ['conanfile.py', 'conanmanifest.txt', "src/data.txt"]
+            if reuploaded:
+                expected_exports.append("conan_export.tgz")
+        if mode == "overlap":
+            expected_exports = ['conanfile.py', 'conanmanifest.txt', "src/data.txt", "src/hello.h"]
+            if reuploaded:
+                expected_exports.append("conan_export.tgz")
         if updated:
             expected_exports.append("license.txt")
-        expected_exports = sorted(expected_exports)
-        export_sources_folder = os.path.join(self.export_folder, EXPORT_SOURCES_DIR)
-        # The export folder might contain or not temporary Python files
-        cached = "__pycache__" if six.PY3 else "conanfile.pyc"
-        exports = [f for f in os.listdir(self.export_folder) if f != cached]
-        self.assertEqual(sorted(exports), expected_exports)
-        self.assertFalse(os.path.exists(export_sources_folder))
+
+        self.assertEqual(scan_folder(self.export_folder), sorted(expected_exports))
+        self.assertFalse(os.path.exists(os.path.join(self.export_folder, EXPORT_SOURCES_DIR)))
 
     def _check_export_uploaded_folder(self, mode, export_folder=None):
         if mode == "exports_sources":
-            expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
-                                           'conanmanifest.txt', EXPORT_SOURCES_TGZ_NAME])
-            expected_exports_sources = ["hello.h"]
-        if mode == "both":
-            expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
-                                           'conanmanifest.txt', "data.txt", EXPORT_TGZ_NAME,
-                                           EXPORT_SOURCES_TGZ_NAME])
-            expected_exports_sources = ["hello.h"]
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "hello.h"),
+                                'conanfile.py', 'conanmanifest.txt', EXPORT_SOURCES_TGZ_NAME]
         if mode == "exports":
-            expected_pkg_exports = sorted([EXPORT_SOURCES_DIR, 'conanfile.py',
-                                           'conanmanifest.txt', "hello.h", EXPORT_TGZ_NAME])
-            expected_exports_sources = []
+            expected_exports = ["hello.h", 'conanfile.py', 'conanmanifest.txt', EXPORT_TGZ_NAME]
+        if mode == "both":
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "hello.h"),
+                                'conanfile.py', 'conanmanifest.txt', "data.txt",
+                                EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME]
+        if mode == "nested":
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "src/hello.h"),
+                                "src/data.txt", 'conanfile.py', 'conanmanifest.txt',
+                                EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME]
+
+        if mode == "overlap":
+            expected_exports = ["%s/%s" % (EXPORT_SOURCES_DIR, "src/hello.h"),
+                                "%s/%s" % (EXPORT_SOURCES_DIR, "src/data.txt"),
+                                "src/data.txt", "src/hello.h", 'conanfile.py', 'conanmanifest.txt',
+                                EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME]
+
         export_folder = export_folder or self.export_folder
-        export_sources_folder = os.path.join(export_folder, EXPORT_SOURCES_DIR)
-        cached = "__pycache__" if six.PY3 else "conanfile.pyc"
-        exports = [f for f in os.listdir(export_folder) if f != cached]
-        exports_sources = [f for f in os.listdir(export_sources_folder) if f != cached]
-        self.assertEqual(sorted(exports), expected_pkg_exports)
-        self.assertEqual(sorted(exports_sources), expected_exports_sources)
+        self.assertTrue(os.path.exists(os.path.join(export_folder, EXPORT_SOURCES_DIR)))
+        self.assertEqual(scan_folder(export_folder), sorted(expected_exports))
 
     def _check_manifest(self, mode):
         manifest = load(os.path.join(self.client.current_folder,
@@ -160,21 +201,49 @@ def _check_manifest(self, mode):
         elif mode == "exports":
             self.assertIn("hello.h: 5d41402abc4b2a76b9719d911017c592",
                           manifest.splitlines())
-        else:
+        elif mode == "both":
             self.assertIn("data.txt: 8d777f385d3dfec8815d20f7496026dc", manifest.splitlines())
             self.assertIn("%s/hello.h: 5d41402abc4b2a76b9719d911017c592" % EXPORT_SOURCES_DIR,
                           manifest.splitlines())
+        elif mode == "nested":
+            self.assertIn("src/data.txt: 8d777f385d3dfec8815d20f7496026dc",
+                          manifest.splitlines())
+            self.assertIn("%s/src/hello.h: 5d41402abc4b2a76b9719d911017c592" % EXPORT_SOURCES_DIR,
+                          manifest.splitlines())
+        else:
+            assert mode == "overlap"
+            self.assertIn("src/data.txt: 8d777f385d3dfec8815d20f7496026dc",
+                          manifest.splitlines())
+            self.assertIn("src/hello.h: 5d41402abc4b2a76b9719d911017c592",
+                          manifest.splitlines())
+            self.assertIn("%s/src/hello.h: 5d41402abc4b2a76b9719d911017c592" % EXPORT_SOURCES_DIR,
+                          manifest.splitlines())
+            self.assertIn("%s/src/data.txt: 8d777f385d3dfec8815d20f7496026dc" % EXPORT_SOURCES_DIR,
+                          manifest.splitlines())
 
     def _create_code(self, mode):
-        conanfile = conanfile_py if mode == "exports" else conanfile_py.replace("exports",
-                                                                                "exports_sources")
-        if mode == "both":
+        if mode == "exports":
+            conanfile = conanfile_py
+        elif mode == "exports_sources":
+            conanfile = conanfile_py.replace("exports", "exports_sources")
+        elif mode == "both":
             conanfile = combined_conanfile
-        self.client.save({"conanfile.py": conanfile,
-                          "hello.h": "hello",
-                          "data.txt": "data"})
+        elif mode == "nested":
+            conanfile = nested_conanfile
+        elif mode == "overlap":
+            conanfile = overlap_conanfile
+
+        if mode in ["nested", "overlap"]:
+            self.client.save({"conanfile.py": conanfile,
+                              "src/hello.h": "hello",
+                              "src/data.txt": "data"})
+        else:
+            self.client.save({"conanfile.py": conanfile,
+                              "hello.h": "hello",
+                              "data.txt": "data"})
 
-    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", ), ("nested", ),
+                           ("overlap", )])
     def copy_test(self, mode):
         # https://github.com/conan-io/conan/issues/943
         self._create_code(mode)
@@ -198,7 +267,8 @@ def copy_test(self, mode):
         self._check_export_uploaded_folder(mode, export_folder)
         self._check_server_folder(mode)
 
-    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", ), ("nested", ),
+                           ("overlap", )])
     def export_test(self, mode):
         self._create_code(mode)
 
@@ -245,7 +315,8 @@ def export_test(self, mode):
         self._check_package_folder(mode)
         self._check_manifest(mode)
 
-    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", ), ("nested", ),
+                           ("overlap", )])
     def export_upload_test(self, mode):
         self._create_code(mode)
 
@@ -273,7 +344,8 @@ def export_upload_test(self, mode):
         self._check_package_folder(mode)
         self._check_manifest(mode)
 
-    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", ), ("nested", ),
+                           ("overlap", )])
     def reupload_test(self, mode):
         """ try to reupload to same and other remote
         """
@@ -294,7 +366,8 @@ def reupload_test(self, mode):
         self._check_export_uploaded_folder(mode)
         self._check_server_folder(mode, self.other_server)
 
-    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", )])
+    @parameterized.expand([("exports", ), ("exports_sources", ), ("both", ), ("nested", ),
+                           ("overlap", )])
     def update_test(self, mode):
         self._create_code(mode)
 

conans/test/utils/test_files.py

@@ -50,9 +50,12 @@ def uncompress_packaged_files(paths, package_reference):
 
 def scan_folder(folder):
     scanned_files = []
-    for root, _, files in os.walk(folder):
+    for root, dirs, files in os.walk(folder):
+        dirs[:] = [d for d in dirs if d != "__pycache__"]
         relative_path = os.path.relpath(root, folder)
         for f in files:
+            if f.endswith(".pyc"):
+                continue
             relative_name = os.path.normpath(os.path.join(relative_path, f)).replace("\\", "/")
             scanned_files.append(relative_name)