From d08e127e19e75e937a663125aa2bae7584c17ef3 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 29 Aug 2017 18:37:06 +0200 Subject: [PATCH 01/39] Rename confusing deps_cpp_info to cpp_info (#1678) * Rename confusing deps_cpp_info to cpp_info * Added pylintrc to the default conan.conf template --- conans/client/conf/__init__.py | 1 + conans/client/generators/cmake.py | 38 ++++++++++++------------ conans/client/generators/qbs.py | 22 +++++++------- conans/client/generators/qmake.py | 30 +++++++++---------- conans/client/generators/text.py | 26 ++++++++-------- conans/client/generators/visualstudio.py | 4 +-- conans/client/importer.py | 4 +-- 7 files changed, 63 insertions(+), 62 deletions(-) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index a6b0794a330..58bd92aaa3c 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -62,6 +62,7 @@ sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO # bash_path = "" # environment CONAN_BASH_PATH (only windows) # recipe_linter = False # environment CONAN_RECIPE_LINTER +# pylintrc = path/to/pylintrc_file # environment CONAN_PYLINTRC # cmake_generator # environment CONAN_CMAKE_GENERATOR # http://www.vtk.org/Wiki/CMake_Cross_Compiling diff --git a/conans/client/generators/cmake.py b/conans/client/generators/cmake.py index c9712b3505d..07e007aa302 100644 --- a/conans/client/generators/cmake.py +++ b/conans/client/generators/cmake.py @@ -1,4 +1,5 @@ from conans.model import Generator +from conans.model.build_info import CppInfo from conans.paths import BUILD_INFO_CMAKE from conans.client.generators.cmake_common import cmake_dependency_vars,\ cmake_macros, generate_targets_section, cmake_dependencies, cmake_package_info,\ @@ -6,35 +7,34 @@ class DepsCppCmake(object): - def __init__(self, deps_cpp_info): - + def __init__(self, cpp_info): def multiline(field): return "\n\t\t\t".join('"%s"' % p.replace("\\", "/") for p in field) - self.include_paths = multiline(deps_cpp_info.include_paths) - self.lib_paths = multiline(deps_cpp_info.lib_paths) - self.res_paths = multiline(deps_cpp_info.res_paths) - self.bin_paths = multiline(deps_cpp_info.bin_paths) - self.build_paths = multiline(deps_cpp_info.build_paths) + self.include_paths = multiline(cpp_info.include_paths) + self.lib_paths = multiline(cpp_info.lib_paths) + self.res_paths = multiline(cpp_info.res_paths) + self.bin_paths = multiline(cpp_info.bin_paths) + self.build_paths = multiline(cpp_info.build_paths) - self.libs = " ".join(deps_cpp_info.libs) - self.defines = "\n\t\t\t".join("-D%s" % d for d in deps_cpp_info.defines) - self.compile_definitions = "\n\t\t\t".join(deps_cpp_info.defines) + self.libs = " ".join(cpp_info.libs) + self.defines = "\n\t\t\t".join("-D%s" % d for d in cpp_info.defines) + self.compile_definitions = "\n\t\t\t".join(cpp_info.defines) - self.cppflags = " ".join(deps_cpp_info.cppflags) - self.cflags = " ".join(deps_cpp_info.cflags) - self.sharedlinkflags = " ".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = " ".join(deps_cpp_info.exelinkflags) + self.cppflags = " ".join(cpp_info.cppflags) + self.cflags = " ".join(cpp_info.cflags) + self.sharedlinkflags = " ".join(cpp_info.sharedlinkflags) + self.exelinkflags = " ".join(cpp_info.exelinkflags) # For modern CMake targets we need to prepare a list to not # loose the elements in the list by replacing " " with ";". Example "-framework Foundation" # Issue: #1251 - self.cppflags_list = ";".join(deps_cpp_info.cppflags) - self.cflags_list = ";".join(deps_cpp_info.cflags) - self.sharedlinkflags_list = ";".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags_list = ";".join(deps_cpp_info.exelinkflags) + self.cppflags_list = ";".join(cpp_info.cppflags) + self.cflags_list = ";".join(cpp_info.cflags) + self.sharedlinkflags_list = ";".join(cpp_info.sharedlinkflags) + self.exelinkflags_list = ";".join(cpp_info.exelinkflags) - self.rootpath = '"%s"' % deps_cpp_info.rootpath.replace("\\", "/") + self.rootpath = '"%s"' % cpp_info.rootpath.replace("\\", "/") class CMakeGenerator(Generator): diff --git a/conans/client/generators/qbs.py b/conans/client/generators/qbs.py index 1dd9e720ffc..3058288c98f 100644 --- a/conans/client/generators/qbs.py +++ b/conans/client/generators/qbs.py @@ -3,24 +3,24 @@ class DepsCppQbs(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): delimiter = ",\n " self.include_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.include_paths) + for p in cpp_info.include_paths) self.lib_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) - self.libs = delimiter.join('"%s"' % l for l in deps_cpp_info.libs) - self.defines = delimiter.join('"%s"' % d for d in deps_cpp_info.defines) + for p in cpp_info.lib_paths) + self.libs = delimiter.join('"%s"' % l for l in cpp_info.libs) + self.defines = delimiter.join('"%s"' % d for d in cpp_info.defines) self.cppflags = delimiter.join('"%s"' % d - for d in deps_cpp_info.cppflags) - self.cflags = delimiter.join('"%s"' % d for d in deps_cpp_info.cflags) + for d in cpp_info.cppflags) + self.cflags = delimiter.join('"%s"' % d for d in cpp_info.cflags) self.sharedlinkflags = delimiter.join('"%s"' % d - for d in deps_cpp_info.sharedlinkflags) + for d in cpp_info.sharedlinkflags) self.sharedlinkflags += delimiter.join('"%s"' % d - for d in deps_cpp_info.exelinkflags) + for d in cpp_info.exelinkflags) self.bin_paths = delimiter.join('"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.bin_paths) - self.rootpath = '%s' % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.bin_paths) + self.rootpath = '%s' % cpp_info.rootpath.replace("\\", "/") class QbsGenerator(Generator): diff --git a/conans/client/generators/qmake.py b/conans/client/generators/qmake.py index 3800196b05a..f735262f548 100644 --- a/conans/client/generators/qmake.py +++ b/conans/client/generators/qmake.py @@ -3,26 +3,26 @@ class DepsCppQmake(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): def multiline(field): return " \\\n ".join('"%s"' % p.replace("\\", "/") for p in field) - self.include_paths = multiline(deps_cpp_info.include_paths) + self.include_paths = multiline(cpp_info.include_paths) self.lib_paths = " \\\n ".join('-L"%s"' % p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) - self.bin_paths = multiline(deps_cpp_info.bin_paths) - self.res_paths = multiline(deps_cpp_info.res_paths) - self.build_paths = multiline(deps_cpp_info.build_paths) - - self.libs = " ".join('-l%s' % l for l in deps_cpp_info.libs) - self.defines = " \\\n ".join('"%s"' % d for d in deps_cpp_info.defines) - self.cppflags = " ".join(deps_cpp_info.cppflags) - self.cflags = " ".join(deps_cpp_info.cflags) - self.sharedlinkflags = " ".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = " ".join(deps_cpp_info.exelinkflags) - - self.rootpath = '%s' % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.lib_paths) + self.bin_paths = multiline(cpp_info.bin_paths) + self.res_paths = multiline(cpp_info.res_paths) + self.build_paths = multiline(cpp_info.build_paths) + + self.libs = " ".join('-l%s' % l for l in cpp_info.libs) + self.defines = " \\\n ".join('"%s"' % d for d in cpp_info.defines) + self.cppflags = " ".join(cpp_info.cppflags) + self.cflags = " ".join(cpp_info.cflags) + self.sharedlinkflags = " ".join(cpp_info.sharedlinkflags) + self.exelinkflags = " ".join(cpp_info.exelinkflags) + + self.rootpath = '%s' % cpp_info.rootpath.replace("\\", "/") class QmakeGenerator(Generator): diff --git a/conans/client/generators/text.py b/conans/client/generators/text.py index 44adeeb55b7..83a9986c500 100644 --- a/conans/client/generators/text.py +++ b/conans/client/generators/text.py @@ -10,24 +10,24 @@ class DepsCppTXT(object): - def __init__(self, deps_cpp_info): + def __init__(self, cpp_info): self.include_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.include_paths) + for p in cpp_info.include_paths) self.lib_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.lib_paths) + for p in cpp_info.lib_paths) self.res_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.res_paths) + for p in cpp_info.res_paths) self.build_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.build_paths) - self.libs = "\n".join(deps_cpp_info.libs) - self.defines = "\n".join(deps_cpp_info.defines) - self.cppflags = "\n".join(deps_cpp_info.cppflags) - self.cflags = "\n".join(deps_cpp_info.cflags) - self.sharedlinkflags = "\n".join(deps_cpp_info.sharedlinkflags) - self.exelinkflags = "\n".join(deps_cpp_info.exelinkflags) + for p in cpp_info.build_paths) + self.libs = "\n".join(cpp_info.libs) + self.defines = "\n".join(cpp_info.defines) + self.cppflags = "\n".join(cpp_info.cppflags) + self.cflags = "\n".join(cpp_info.cflags) + self.sharedlinkflags = "\n".join(cpp_info.sharedlinkflags) + self.exelinkflags = "\n".join(cpp_info.exelinkflags) self.bin_paths = "\n".join(p.replace("\\", "/") - for p in deps_cpp_info.bin_paths) - self.rootpath = "%s" % deps_cpp_info.rootpath.replace("\\", "/") + for p in cpp_info.bin_paths) + self.rootpath = "%s" % cpp_info.rootpath.replace("\\", "/") class TXTGenerator(Generator): diff --git a/conans/client/generators/visualstudio.py b/conans/client/generators/visualstudio.py index 35eb7ff3ae1..35d576da5d2 100644 --- a/conans/client/generators/visualstudio.py +++ b/conans/client/generators/visualstudio.py @@ -38,9 +38,9 @@ class VisualStudioGenerator(Generator): def _format_items(self): sections = [] - for dep_name, dep_cpp_info in self.deps_build_info.dependencies: + for dep_name, cpp_info in self.deps_build_info.dependencies: fields = { - 'root_dir': dep_cpp_info.rootpath, + 'root_dir': cpp_info.rootpath, 'name': dep_name.replace(".", "-") } section = self.item_template.format(**fields) diff --git a/conans/client/importer.py b/conans/client/importer.py index bf6767a44f4..78b39f3ff30 100644 --- a/conans/client/importer.py +++ b/conans/client/importer.py @@ -112,6 +112,6 @@ def _get_folders(self, pattern): each dependency """ if not pattern: - return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies} - return {pkg: deps.rootpath for pkg, deps in self._conanfile.deps_cpp_info.dependencies + return {pkg: cpp_info.rootpath for pkg, cpp_info in self._conanfile.deps_cpp_info.dependencies} + return {pkg: cpp_info.rootpath for pkg, cpp_info in self._conanfile.deps_cpp_info.dependencies if fnmatch.fnmatch(pkg, pattern)} From e901384d6d6eb0606f7e97df7b37f31cd1d1da41 Mon Sep 17 00:00:00 2001 From: Mark Browning Date: Thu, 31 Aug 2017 04:08:24 -0500 Subject: [PATCH 02/39] Add ?:"Any authenticated user" to BasicAuthorizer (#1687) * Add ?:"Any authenticated user" to BasicAuthorizer * Add BasicAuthorizer '?' wildcard rule unit tests * Add '?' snippet example to default_server_conf * Correct authorizer unit tests and add more for '?' --- conans/server/conf/default_server_conf.py | 11 +++-- conans/server/service/authorize.py | 5 ++- conans/test/server/service/authorizer_test.py | 43 ++++++++++++++++++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/conans/server/conf/default_server_conf.py b/conans/server/conf/default_server_conf.py index b846e597e9d..6b3f232d51e 100644 --- a/conans/server/conf/default_server_conf.py +++ b/conans/server/conf/default_server_conf.py @@ -45,10 +45,15 @@ # name/version@user/channel: user1, user2, user3 # The rules are applied in order. If a rule applies to a conan, system wont look further. # -# Example: All versions of opencv package from lasote user in testing channel is only -# readable by default_user and default_user2. Rest of packages are world readable +# Example: +# All versions of opencv package from lasote user in testing channel are only +# readable by default_user and default_user2. +# All versions of internal package from any user/channel are only readable by +# authenticated users. +# Rest of packages are world readable. # -# opencv/1.2.3@lasote/testing: default_user default_user2 +# opencv/*@lasote/testing: default_user default_user2 +# internal/*@*/*: ? # *:*@*/*: * # # By default all users can read all blocks diff --git a/conans/server/service/authorize.py b/conans/server/service/authorize.py index 7223729277e..fb531661f4c 100644 --- a/conans/server/service/authorize.py +++ b/conans/server/service/authorize.py @@ -189,7 +189,10 @@ def _check_rule_ok(self, username, rule, conan_reference): return True # Ok, applies and match username else: if username: - raise ForbiddenException("Permission denied") + if authorized_users[0] == "?": + return True #Ok, applies and match any authenticated username + else: + raise ForbiddenException("Permission denied") else: raise AuthenticationException() diff --git a/conans/test/server/service/authorizer_test.py b/conans/test/server/service/authorizer_test.py index 994333b204a..7b0652667e6 100644 --- a/conans/test/server/service/authorizer_test.py +++ b/conans/test/server/service/authorizer_test.py @@ -1,6 +1,6 @@ import unittest from conans.server.service.authorize import BasicAuthorizer -from conans.errors import ForbiddenException, InternalErrorException +from conans.errors import ForbiddenException, InternalErrorException, AuthenticationException from conans.model.ref import ConanFileReference, PackageReference @@ -135,6 +135,47 @@ def permissions_test(self): # Pepe can't write other package self.assertRaises(ForbiddenException, authorizer.check_write_package, "pepe", self.package_reference2) + + def authenticated_user_wildcard_permissions_test(self): + """Check that authenciated user wildcard permissions logic is ok""" + # Only authenticated users can read openssl + read_perms = [(str(self.openssl_ref), "?"), ("*/*@*/*", "*")] + # Authenticated users can write any + write_perms = [("*/*@*/*", "?")] + + authorizer = BasicAuthorizer(read_perms, write_perms) + + # READ PERMISSIONS + + # Authenticated user can read conan + authorizer.check_read_conan("pepe", self.openssl_ref) + + # Authenticated user can read package + authorizer.check_read_package("pepe", self.package_reference) + + # Anonymous user can not read conan, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_read_conan, None, self.openssl_ref) + + # Anonymous user can not read package, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_read_package, None, self.package_reference) + + # WRITE PERMISSIONS + + # Authenticated user can write conan + authorizer.check_write_conan("pepe", self.openssl_ref) + + # Authenticated user can write package + authorizer.check_write_package("pepe", self.package_reference) + + # Anonymous user can not write conan, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_write_conan, None, self.openssl_ref) + + # Anonymous user can not write package, they must authenticate + self.assertRaises(AuthenticationException, + authorizer.check_write_package, None, self.package_reference) def users_test(self): """Check that lists of user names are parsed correctly""" From b1b429f560bbb35c84af82f27120299422152b95 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Thu, 31 Aug 2017 16:53:44 +0200 Subject: [PATCH 03/39] Dev version develop --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index 75c5e8e485e..74732003bd1 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,5 +16,5 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.26.0' +__version__ = '0.27.0-dev' From 4ae5d5d20c8d9638c873fc0c611560b051b73a53 Mon Sep 17 00:00:00 2001 From: Ivan Bobev Date: Mon, 4 Sep 2017 00:40:46 +0300 Subject: [PATCH 04/39] Add option to pass different build type in CMake build helper. (#1686) * Add option to pass different build type in CMake build helper. Add additional argument build_type for CMake build helper build, install and test methods to be able to override Conan build_type setting. * Add build_type argument to CMake build helper Add new build_type argument to Conan CMake build helper which is used for overriding of the default build_type comming from recipe settings. * Revert "Add option to pass different build type in CMake build helper." This reverts commit 8f8b53cc5ea555414b54a6bf17c1ce040bb827e1. * Add build_type propery to CMake build helper class - Also fix bug in runtime property which uses _defs_to_string as method of CMake build helper class, but now it is module free function. * Added test and fixed definitions --- conans/client/cmake.py | 28 ++++++++++++----- conans/test/functional/cmake_test.py | 45 +++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 8 deletions(-) diff --git a/conans/client/cmake.py b/conans/client/cmake.py index eaf556cc597..646d9e37398 100644 --- a/conans/client/cmake.py +++ b/conans/client/cmake.py @@ -39,13 +39,14 @@ def _get_env_cmake_system_name(): class CMake(object): def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True, - parallel=True): + parallel=True, build_type=None): """ :param settings_or_conanfile: Conanfile instance (or settings for retro compatibility) :param generator: Generator name to use or none to autodetect :param cmake_system_name: False to not use CMAKE_SYSTEM_NAME variable, True for auto-detect or directly a string with the system name :param parallel: Try to build with multiple cores if available + :param build_type: Overrides default build type comming from settings """ if isinstance(settings_or_conanfile, Settings): self._settings = settings_or_conanfile @@ -64,10 +65,10 @@ def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True self._compiler = self._settings.get_safe("compiler") self._compiler_version = self._settings.get_safe("compiler.version") self._arch = self._settings.get_safe("arch") - self._build_type = self._settings.get_safe("build_type") self._op_system_version = self._settings.get_safe("os.version") self._libcxx = self._settings.get_safe("compiler.libcxx") self._runtime = self._settings.get_safe("compiler.runtime") + self._build_type = self._settings.get_safe("build_type") self.generator = generator or self._generator() self.build_dir = None @@ -76,6 +77,23 @@ def __init__(self, settings_or_conanfile, generator=None, cmake_system_name=True self._cmake_system_name = cmake_system_name self.parallel = parallel self.definitions = self._get_cmake_definitions() + if build_type and build_type != self._build_type: + # Call the setter to warn and update the definitions if needed + self.build_type = build_type + + @property + def build_type(self): + return self._build_type + + @build_type.setter + def build_type(self, build_type): + settings_build_type = self._settings.get_safe("build_type") + if build_type != settings_build_type: + self._conanfile.output.warn( + 'Set CMake build type "%s" is different than the settings build_type "%s"' + % (build_type, settings_build_type)) + self._build_type = build_type + self.definitions.update(self._build_type_definition()) @property def flags(self): @@ -213,10 +231,6 @@ def command_line(self): '-Wno-dev' ]) - @property - def build_type(self): - return self._defs_to_string(self._build_type_definition()) - def _build_type_definition(self): if self._build_type and not self.is_multi_configuration: return {'CMAKE_BUILD_TYPE': self._build_type} @@ -224,7 +238,7 @@ def _build_type_definition(self): @property def runtime(self): - return self._defs_to_string(self._runtime_definition()) + return _defs_to_string(self._runtime_definition()) def _runtime_definition(self): if self._runtime: diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index 30a0f16b3e6..b7fdf9726e7 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -12,6 +12,7 @@ from conans.model.settings import Settings from conans.client.conf import default_settings_yml from conans.client.cmake import CMake +from conans.test.utils.tools import TestBufferConanOutput from conans.tools import cpu_count from conans.util.files import save @@ -23,6 +24,48 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.tempdir) + def build_type_ovewrite_test(self): + settings = Settings.loads(default_settings_yml) + settings.os = "Linux" + settings.compiler = "gcc" + settings.compiler.version = "6.3" + settings.arch = "x86" + settings.build_type = "Release" + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + cmake.build_type = "Debug" + self.assertIn('WARN: Set CMake build type "Debug" is different than the ' + 'settings build_type "Release"', conan_file.output) + self.assertEquals(cmake.build_type, "Debug") + self.assertIn('-DCMAKE_BUILD_TYPE="Debug"', cmake.command_line) + + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + self.assertNotIn('WARN: Set CMake build type ', conan_file.output) + self.assertEquals(cmake.build_type, "Release") + + # Now with visual, (multiconfig) + settings = Settings.loads(default_settings_yml) + settings.os = "Windows" + settings.compiler = "Visual Studio" + settings.compiler.version = "15" + settings.arch = "x86" + settings.build_type = "Release" + conan_file = ConanFileMock() + conan_file.settings = settings + cmake = CMake(conan_file) + cmake.build_type = "Debug" + self.assertIn('WARN: Set CMake build type "Debug" is different than the ' + 'settings build_type "Release"', conan_file.output) + self.assertEquals(cmake.build_type, "Debug") + self.assertNotIn('-DCMAKE_BUILD_TYPE="Debug"', cmake.command_line) + self.assertIn("--config Debug", cmake.build_config) + cmake = CMake(conan_file) + cmake.build_type = "Release" + self.assertIn("--config Release", cmake.build_config) + def loads_default_test(self): settings = Settings.loads(default_settings_yml) settings.os = "Windows" @@ -420,7 +463,7 @@ def __init__(self, shared=None): self.source_folder = self.build_folder = "." self.settings = None self.deps_cpp_info = namedtuple("deps_cpp_info", "sysroot")("/path/to/sysroot") - self.output = namedtuple("output", "warn")(lambda x: x) + self.output = TestBufferConanOutput() if shared is not None: self.options = namedtuple("options", "shared")(shared) From 59ac69e34967bd87498cb37e5c95e65bf55f0471 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 4 Sep 2017 08:44:06 +0200 Subject: [PATCH 05/39] minor style fixes (#1698) --- conans/client/command.py | 7 +++---- conans/client/new_ci.py | 7 +++++-- conans/errors.py | 4 ++-- .../test/{functional => command}/conan_get_test.py | 14 +++++++++----- conans/test/command/config_test.py | 9 +++------ conans/test/command/create_test.py | 2 +- conans/test/integration/profile_test.py | 3 --- 7 files changed, 23 insertions(+), 23 deletions(-) rename conans/test/{functional => command}/conan_get_test.py (89%) diff --git a/conans/client/command.py b/conans/client/command.py index 6f09f9cd504..21effb78e05 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -368,11 +368,11 @@ def info(self, *args): only = [] if only and args.paths and (set(only) - set(path_only_options)): raise ConanException("Invalid --only value '%s' with --path specified, allowed values: [%s]." - % (only, str_path_only_options)) + % (only, str_path_only_options)) elif only and not args.paths and (set(only) - set(info_only_options)): raise ConanException("Invalid --only value '%s', allowed values: [%s].\n" - "Use --only=None to show only the references." % - (only, str_only_options)) + "Use --only=None to show only the references." + % (only, str_only_options)) if args.graph: self._outputer.info_graph(args.graph, deps_graph, project_reference, args.cwd) @@ -712,7 +712,6 @@ def remote(self, *args): verify_ssl = get_bool_from_text(args.verify_ssl) if hasattr(args, 'verify_ssl') else False - remote = args.remote if hasattr(args, 'remote') else None url = args.url if hasattr(args, 'url') else None diff --git a/conans/client/new_ci.py b/conans/client/new_ci.py index 0fba66bf964..2dc8c80c474 100644 --- a/conans/client/new_ci.py +++ b/conans/client/new_ci.py @@ -190,6 +190,7 @@ def get_travis(name, version, user, channel, linux_gcc_versions, linux_clang_ver ".travis/run.sh": travis_run} return files + def get_appveyor(name, version, user, channel, visual_versions, upload_url): config = [] visual_config = """ - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio {image} @@ -205,6 +206,7 @@ def get_appveyor(name, version, user, channel, visual_versions, upload_url): channel=channel, configs=configs, upload=upload)} return files + def get_gitlab(name, version, user, channel, linux_gcc_versions, linux_clang_versions, upload_url): config = [] @@ -219,13 +221,14 @@ def get_gitlab(name, version, user, channel, linux_gcc_versions, linux_clang_ver configs = "".join(config) upload = ('CONAN_UPLOAD: "%s"\n' % upload_url) if upload_url else "" files = {".gitlab-ci.yml": gitlab.format(name=name, version=version, user=user, channel=channel, - configs=configs, upload=upload)} + configs=configs, upload=upload)} return files + def ci_get_files(name, version, user, channel, visual_versions, linux_gcc_versions, linux_clang_versions, osx_clang_versions, shared, upload_url, gitlab_gcc_versions, gitlab_clang_versions): if shared and not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions or - gitlab_gcc_versions or gitlab_clang_versions): + gitlab_gcc_versions or gitlab_clang_versions): raise ConanException("Trying to specify 'shared' in CI, but no CI system specified") if not (visual_versions or linux_gcc_versions or linux_clang_versions or osx_clang_versions or gitlab_gcc_versions or gitlab_clang_versions): diff --git a/conans/errors.py b/conans/errors.py index 66d717dc436..70cf7327d05 100644 --- a/conans/errors.py +++ b/conans/errors.py @@ -35,7 +35,7 @@ def _format_conanfile_exception(scope, method, exception): index = -1 while True: # If out of index will raise and will be captured later filepath, line, name, contents = traceback.extract_tb(tb, 40)[index] # 40 levels of nested functions max, get the latest - if not "conanfile.py" in filepath: # Avoid show trace from internal conan source code + if "conanfile.py" not in filepath: # Avoid show trace from internal conan source code index -= 1 else: break @@ -70,8 +70,8 @@ class ConanOutdatedClient(ConanException): class ConanExceptionInUserConanfileMethod(ConanException): pass -# Remote exceptions # +# Remote exceptions # class InternalErrorException(ConanException): """ Generic 500 error diff --git a/conans/test/functional/conan_get_test.py b/conans/test/command/conan_get_test.py similarity index 89% rename from conans/test/functional/conan_get_test.py rename to conans/test/command/conan_get_test.py index edc5ffd41fd..a5c74c5d6ba 100644 --- a/conans/test/functional/conan_get_test.py +++ b/conans/test/command/conan_get_test.py @@ -87,7 +87,8 @@ def test_get_remote(self): # Remote search, dir list self.client.run('get Hello0/0.1@lasote/channel . -r default --raw') - self.assertIn("conan_export.tgz\nconan_sources.tgz\nconanfile.py\nconanmanifest.txt", self.client.user_io.out) + self.assertIn("conan_export.tgz\nconan_sources.tgz\nconanfile.py\nconanmanifest.txt", + self.client.user_io.out) # Remote search, conanfile print self.client.run('get Hello0/0.1@lasote/channel -r default --raw') @@ -95,12 +96,15 @@ def test_get_remote(self): # List package dir self.client.run('get Hello0/0.1@lasote/channel "." -p 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9 --raw -r default') - self.assertEquals("conan_package.tgz\nconaninfo.txt\nconanmanifest.txt\n", self.client.user_io.out) + self.assertEquals("conan_package.tgz\nconaninfo.txt\nconanmanifest.txt\n", + self.client.user_io.out) def test_not_found(self): self.client.run('get Hello0/0.1@lasote/channel "." -r default', ignore_error=True) self.assertIn("Recipe Hello0/0.1@lasote/channel not found", self.client.user_io.out) - self.client.run('get Hello0/0.1@lasote/channel "." -r default -p 123123123123123', ignore_error=True) - self.assertIn("Package Hello0/0.1@lasote/channel:123123123123123 not found", self.client.user_io.out) - + error = self.client.run('get Hello0/0.1@lasote/channel "." -r default -p 123123123123123', + ignore_error=True) + self.assertTrue(error) + self.assertIn("Package Hello0/0.1@lasote/channel:123123123123123 not found", + self.client.user_io.out) diff --git a/conans/test/command/config_test.py b/conans/test/command/config_test.py index e0bd1ded2d9..2d6d64c77aa 100644 --- a/conans/test/command/config_test.py +++ b/conans/test/command/config_test.py @@ -1,10 +1,7 @@ -from conans.test.utils.tools import TestClient import unittest -from conans.util.files import save, load -from conans.client.conf import default_client_conf -from conans import tools -from conans.test.utils.test_files import temp_folder -import os + +from conans.util.files import load +from conans.test.utils.tools import TestClient class ConfigTest(unittest.TestCase): diff --git a/conans/test/command/create_test.py b/conans/test/command/create_test.py index c82ab682f88..af041a01fd2 100644 --- a/conans/test/command/create_test.py +++ b/conans/test/command/create_test.py @@ -164,7 +164,7 @@ def build(self): [build_requires] BuildRequire/0.1@conan/stable ''', -"test_package/conanfile.py": """from conans import ConanFile + "test_package/conanfile.py": """from conans import ConanFile import os class MyTest(ConanFile): diff --git a/conans/test/integration/profile_test.py b/conans/test/integration/profile_test.py index d5f4a771661..d19cb719046 100644 --- a/conans/test/integration/profile_test.py +++ b/conans/test/integration/profile_test.py @@ -1,8 +1,5 @@ import unittest -from conans.client.profile_loader import _load_profile -from conans.model.env_info import EnvValues - from conans.test.utils.tools import TestClient from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.util.files import save, load From 5d77c05ed67d0d78493f8a675170f1795b1f5dca Mon Sep 17 00:00:00 2001 From: SSE4 Date: Wed, 6 Sep 2017 03:55:47 +0700 Subject: [PATCH 06/39] replace_in_file displays a warning if search pattern wasn't found (#1709) * replace_in_file displays a warning if search pattern wasn't found optionally, if strict flag is given, it raises exception * - make strict=True default --- conans/tools.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conans/tools.py b/conans/tools.py index f6ce34c77fa..c84f2aba9f8 100644 --- a/conans/tools.py +++ b/conans/tools.py @@ -368,8 +368,14 @@ def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): # save(filename, content) -def replace_in_file(file_path, search, replace): +def replace_in_file(file_path, search, replace, strict=True): content = load(file_path) + if -1 == content.find(search): + message = "replace_in_file didn't find pattern '%s' in '%s' file." % (search, file_path) + if strict: + raise ConanException(message) + else: + _global_output.warn(message) content = content.replace(search, replace) content = content.encode("utf-8") with open(file_path, "wb") as handle: From 8a2d93b04a13ab455db6f3bb5ca647f93e1a6baf Mon Sep 17 00:00:00 2001 From: James Date: Fri, 8 Sep 2017 09:51:31 +0200 Subject: [PATCH 07/39] Filter directories (#1719) * #685 remove trailing slash from the remote URL Signed-off-by: SSE4 * - filter out non-existing and empty directories * fixed tests --- conans/model/build_info.py | 35 +++++++++++++------ conans/test/command/build_test.py | 5 +++ conans/test/integration/conan_env_test.py | 4 +++ .../integration/flat_requirements_test.py | 8 +++++ conans/test/model/build_info_test.py | 24 +++++++++---- 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/conans/model/build_info.py b/conans/model/build_info.py index 92133bb5995..fed762aa936 100644 --- a/conans/model/build_info.py +++ b/conans/model/build_info.py @@ -27,31 +27,46 @@ def __init__(self): self.exelinkflags = [] # linker flags self.rootpath = "" self.sysroot = None + self._include_paths = None + self._lib_paths = None + self._bin_paths = None + self._build_paths = None + self._res_paths = None + + def _filter_paths(self, paths): + abs_paths = [os.path.join(self.rootpath, p) + if not os.path.isabs(p) else p for p in paths] + return [p for p in abs_paths if os.path.isdir(p)] @property def include_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.includedirs] + if self._include_paths is None: + self._include_paths = self._filter_paths(self.includedirs) + return self._include_paths @property def lib_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.libdirs] + if self._lib_paths is None: + self._lib_paths = self._filter_paths(self.libdirs) + return self._lib_paths @property def bin_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.bindirs] + if self._bin_paths is None: + self._bin_paths = self._filter_paths(self.bindirs) + return self._bin_paths @property def build_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.builddirs] + if self._build_paths is None: + self._build_paths = self._filter_paths(self.builddirs) + return self._build_paths @property def res_paths(self): - return [os.path.join(self.rootpath, p) - if not os.path.isabs(p) else p for p in self.resdirs] + if self._res_paths is None: + self._res_paths = self._filter_paths(self.resdirs) + return self._res_paths class CppInfo(_CppInfo): diff --git a/conans/test/command/build_test.py b/conans/test/command/build_test.py index 18eb7fbbfbe..e0da0f1b0bf 100644 --- a/conans/test/command/build_test.py +++ b/conans/test/command/build_test.py @@ -21,10 +21,15 @@ def build(self): conanfile_dep = """ from conans import ConanFile +from conans.tools import mkdir +import os class AConan(ConanFile): name = "Hello" version = "0.1" + + def package(self): + mkdir(os.path.join(self.package_folder, "include")) """ diff --git a/conans/test/integration/conan_env_test.py b/conans/test/integration/conan_env_test.py index d907ea0ddad..5031a6458e7 100644 --- a/conans/test/integration/conan_env_test.py +++ b/conans/test/integration/conan_env_test.py @@ -264,6 +264,8 @@ def test_run_env(self): client = TestClient() conanfile = ''' from conans import ConanFile +from conans.tools import mkdir +import os class HelloConan(ConanFile): name = "Hello" @@ -271,6 +273,8 @@ class HelloConan(ConanFile): build_policy = "missing" def package_info(self): + mkdir(os.path.join(self.package_folder, "bin2")) + mkdir(os.path.join(self.package_folder, "lib2")) self.cpp_info.bindirs.append("bin2") self.cpp_info.libdirs.append("lib2") diff --git a/conans/test/integration/flat_requirements_test.py b/conans/test/integration/flat_requirements_test.py index c1e5c4463d0..98a652f9379 100644 --- a/conans/test/integration/flat_requirements_test.py +++ b/conans/test/integration/flat_requirements_test.py @@ -15,6 +15,14 @@ def setUp(self): self.conan_reference = ConanFileReference.loads("Hello0/0.1@lasote/stable") self.files = cpp_hello_conan_files("Hello0", "0.1", build=False) self.conan = TestClient() + package = """def package(self): + import os + os.mkdir(os.path.join(self.package_folder, "include")) + os.mkdir(os.path.join(self.package_folder, "lib")) + os.mkdir(os.path.join(self.package_folder, "bin")) +""" + self.files["conanfile.py"] = self.files["conanfile.py"].replace("def package(self):", + package) self.conan.save(self.files) self.conan.run("export lasote/stable") diff --git a/conans/test/model/build_info_test.py b/conans/test/model/build_info_test.py index 0d64ee0c32a..15e0a8e3f26 100644 --- a/conans/test/model/build_info_test.py +++ b/conans/test/model/build_info_test.py @@ -6,6 +6,7 @@ from conans.model.env_info import DepsEnvInfo from conans.test.utils.test_files import temp_folder import platform +from conans.util.files import mkdir class BuildInfoTest(unittest.TestCase): @@ -102,13 +103,22 @@ def configs_test(self): def cpp_info_test(self): folder = temp_folder() + mkdir(os.path.join(folder, "include")) + mkdir(os.path.join(folder, "lib")) + mkdir(os.path.join(folder, "local_bindir")) + abs_folder = temp_folder() + abs_include = os.path.join(abs_folder, "usr/include") + abs_lib = os.path.join(abs_folder, "usr/lib") + abs_bin = os.path.join(abs_folder, "usr/bin") + mkdir(abs_include) + mkdir(abs_lib) + mkdir(abs_bin) info = CppInfo(folder) - info.includedirs.append("/usr/include") - info.libdirs.append("/usr/lib") - bin_abs_dir = "C:/usr/bin" if platform.system() == "Windows" else "/tmp" - info.bindirs.append(bin_abs_dir) + info.includedirs.append(abs_include) + info.libdirs.append(abs_lib) + info.bindirs.append(abs_bin) info.bindirs.append("local_bindir") - self.assertEqual(info.include_paths, [os.path.join(folder, "include"), "/usr/include"]) - self.assertEqual(info.lib_paths, [os.path.join(folder, "lib"), "/usr/lib"]) - self.assertEqual(info.bin_paths, [os.path.join(folder, "bin"), bin_abs_dir, + self.assertEqual(info.include_paths, [os.path.join(folder, "include"), abs_include]) + self.assertEqual(info.lib_paths, [os.path.join(folder, "lib"), abs_lib]) + self.assertEqual(info.bin_paths, [abs_bin, os.path.join(folder, "local_bindir")]) From 9cc51b21af805bb6798e67244b64ab03f2c4238f Mon Sep 17 00:00:00 2001 From: James Date: Mon, 11 Sep 2017 12:54:10 +0200 Subject: [PATCH 08/39] reverting to cmake 3.7 (#1737) * reverting to cmake 3.7 * changed verbosity --- .ci/appveyor/install.bat | 14 ++++++++++++-- .ci/appveyor/test.bat | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.ci/appveyor/install.bat b/.ci/appveyor/install.bat index 15a61ee8036..9548bc0e2c9 100644 --- a/.ci/appveyor/install.bat +++ b/.ci/appveyor/install.bat @@ -1,10 +1,20 @@ if not exist "C:\mingw64" appveyor DownloadFile "https://s3-eu-west-1.amazonaws.com/downloads.conan.io/x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z" if not exist "C:\mingw64" 7z x x86_64-6.3.0-release-posix-sjlj-rt_v5-rev1.7z -oc:\ + +set CMAKE_URL="https://cmake.org/files/v3.7/cmake-3.7.0-win64-x64.zip" +mkdir C:\projects\deps +SET ORIGINAL_DIR=%CD% +cd C:\projects\deps +appveyor DownloadFile %CMAKE_URL% -FileName cmake.zip +7z x cmake.zip -oC:\projects\deps > nul +move C:\projects\deps\cmake-* C:\projects\deps\cmake +set PATH=C:\projects\deps\cmake\bin;%PATH% +cmake --version +cd %ORIGINAL_DIR% + SET PATH=%PYTHON%;%PYTHON%\\Scripts;C:\\mingw64\\bin;%PATH% SET PYTHONPATH=%PYTHONPATH%;%CD% SET CONAN_LOGGING_LEVEL=10 -SET CONAN_COMPILER=Visual Studio -SET CONAN_COMPILER_VERSION=12 %PYTHON%/Scripts/pip.exe install -r conans/requirements.txt %PYTHON%/Scripts/pip.exe install -r conans/requirements_dev.txt %PYTHON%/Scripts/pip.exe install -r conans/requirements_server.txt diff --git a/.ci/appveyor/test.bat b/.ci/appveyor/test.bat index c501810b452..f72fb740f75 100644 --- a/.ci/appveyor/test.bat +++ b/.ci/appveyor/test.bat @@ -1 +1 @@ -nosetests --with-coverage conans.test +nosetests --with-coverage --verbosity=2 conans.test From 5709f6003d9df922a9d8c66aa2c53743ebae8cc8 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 11 Sep 2017 14:25:28 +0200 Subject: [PATCH 09/39] Feature/dirty files fixes (#1720) * Dirty file outside the source folder and not generated by local method * untouched file * Do not copy recipe and manifest to the source folder * Fixed test --- conans/client/export.py | 8 +++---- conans/client/installer.py | 8 ++++--- conans/client/manager.py | 3 ++- conans/client/source.py | 23 ++++++++----------- conans/paths.py | 6 ++++- conans/test/command/source_test.py | 3 ++- conans/test/integration/build_id_test.py | 10 ++++---- .../test/integration/export_sources_test.py | 4 ++-- conans/test/integration/go_complete_test.py | 7 +++--- 9 files changed, 38 insertions(+), 34 deletions(-) diff --git a/conans/client/export.py b/conans/client/export.py index 7cbe66326ce..9e79a3795f1 100644 --- a/conans/client/export.py +++ b/conans/client/export.py @@ -5,7 +5,7 @@ import shutil import os from conans.util.files import save, load, rmdir -from conans.paths import CONAN_MANIFEST, CONANFILE, DIRTY_FILE +from conans.paths import CONAN_MANIFEST, CONANFILE from conans.errors import ConanException from conans.model.manifest import FileTreeManifest from conans.client.output import ScopedOutput @@ -73,9 +73,9 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so save(os.path.join(destination_folder, CONAN_MANIFEST), str(digest)) source = paths.source(conan_ref, conanfile.short_paths) - dirty = os.path.join(source, DIRTY_FILE) + dirty_file_path = paths.dirty_sources_file(conan_ref) remove = False - if os.path.exists(dirty): + if os.path.exists(dirty_file_path): output.info("Source folder is dirty, forcing removal") remove = True elif modified_recipe and not keep_source and os.path.exists(source): @@ -91,7 +91,7 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so output.error("Unable to delete source folder. " "Will be marked as dirty for deletion") output.warn(str(e)) - save(os.path.join(source, DIRTY_FILE), "") + save(dirty_file_path, "") def _init_export_folder(destination_folder, destination_src_folder): diff --git a/conans/client/installer.py b/conans/client/installer.py index 6bc4517d7e2..e1221537bf2 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -347,11 +347,12 @@ def _build_conanfile(self, conan_ref, conan_file, package_reference, package_fol src_folder = self._client_cache.source(conan_ref, conan_file.short_paths) export_folder = self._client_cache.export(conan_ref) export_source_folder = self._client_cache.export_sources(conan_ref, conan_file.short_paths) + dirty_file_path = self._client_cache.dirty_sources_file(conan_ref) self._handle_system_requirements(conan_ref, package_reference, conan_file, output) with environment_append(conan_file.env): self._build_package(export_folder, export_source_folder, src_folder, build_folder, - package_folder, conan_file, output) + package_folder, dirty_file_path, conan_file, output) return build_folder def _package_conanfile(self, conan_ref, conan_file, package_reference, build_folder, @@ -424,7 +425,7 @@ def call_system_requirements(self, conan_file, output): raise ConanException("Error in system requirements") def _build_package(self, export_folder, export_source_folder, src_folder, build_folder, - package_folder, conan_file, output): + package_folder, dirty_file_path, conan_file, output): """ builds the package, creating the corresponding build folder if necessary and copying there the contents from the src folder. The code is duplicated in every build, as some configure processes actually change the source @@ -439,7 +440,8 @@ def _build_package(self, export_folder, export_source_folder, src_folder, build_ "Close any app using it, and retry" % str(e)) output.info('Building your package in %s' % build_folder) - config_source(export_folder, export_source_folder, src_folder, conan_file, output) + config_source(export_folder, export_source_folder, src_folder, dirty_file_path, + conan_file, output) output.info('Copying sources to build folder') def check_max_path_len(src, files): diff --git a/conans/client/manager.py b/conans/client/manager.py index 357a6459c42..a0edecfb98a 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -377,10 +377,11 @@ def source(self, current_path, reference, force): conanfile = self.load_consumer_conanfile(conanfile_path, current_path, output, reference=reference, deps_cpp_info_required=None) + dirty_file_path = self._client_cache.dirty_sources_file(reference) src_folder = self._client_cache.source(reference, conanfile.short_paths) export_folder = self._client_cache.export(reference) export_src_folder = self._client_cache.export_sources(reference, conanfile.short_paths) - config_source(export_folder, export_src_folder, src_folder, conanfile, output, force) + config_source(export_folder, export_src_folder, src_folder, dirty_file_path, conanfile, output, force) def imports_undo(self, current_path): undo_imports(current_path, self._user_io.out) diff --git a/conans/client/source.py b/conans/client/source.py index 3b0aa7a0c31..6bf2514cc5a 100644 --- a/conans/client/source.py +++ b/conans/client/source.py @@ -6,7 +6,7 @@ from conans import tools from conans.errors import ConanException, conanfile_exception_formatter, \ ConanExceptionInUserConanfileMethod -from conans.paths import DIRTY_FILE, EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE +from conans.paths import EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE, CONAN_MANIFEST from conans.util.files import rmdir, save @@ -21,18 +21,18 @@ def merge_directories(src, dst): shutil.copy2(src_file, dst_file) -def config_source(export_folder, export_source_folder, src_folder, conan_file, output, force=False): +def config_source(export_folder, export_source_folder, src_folder, dirty_file_path, + conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code """ - dirty = os.path.join(src_folder, DIRTY_FILE) def remove_source(raise_error=True): output.warn("This can take a while for big packages") try: rmdir(src_folder) except BaseException as e_rm: - save(dirty, "") # Creation of DIRTY flag + save(dirty_file_path, "") # Creation of DIRTY flag msg = str(e_rm) if six.PY2: msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1 @@ -44,7 +44,7 @@ def remove_source(raise_error=True): if force: output.warn("Forced removal of source folder") remove_source() - elif os.path.exists(dirty): + elif os.path.exists(dirty_file_path): output.warn("Trying to remove dirty source folder") remove_source() elif conan_file.build_policy_always: @@ -56,7 +56,8 @@ def remove_source(raise_error=True): shutil.copytree(export_folder, src_folder, symlinks=True) # Now move the export-sources to the right location merge_directories(export_source_folder, src_folder) - for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", CONANFILE+"o"): + for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c", + CONANFILE+"o", CONANFILE, CONAN_MANIFEST): try: os.remove(os.path.join(src_folder, f)) except OSError: @@ -66,13 +67,13 @@ def remove_source(raise_error=True): except OSError: pass - save(dirty, "") # Creation of DIRTY flag + save(dirty_file_path, "") # Creation of DIRTY flag os.chdir(src_folder) try: with tools.environment_append(conan_file.env): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.source() - os.remove(dirty) # Everything went well, remove DIRTY flag + os.remove(dirty_file_path) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder @@ -86,16 +87,10 @@ def remove_source(raise_error=True): def config_source_local(current_path, conan_file, output): output.info('Configuring sources in %s' % current_path) - dirty = os.path.join(current_path, DIRTY_FILE) - if os.path.exists(dirty): - output.warn("Your previous source command failed") - - save(dirty, "") # Creation of DIRTY flag try: with conanfile_exception_formatter(str(conan_file), "source"): with tools.environment_append(conan_file.env): conan_file.source() - os.remove(dirty) # Everything went well, remove DIRTY flag except ConanExceptionInUserConanfileMethod: raise except Exception as e: diff --git a/conans/paths.py b/conans/paths.py index 2f6e1324dd5..9f89e342736 100644 --- a/conans/paths.py +++ b/conans/paths.py @@ -29,7 +29,7 @@ CONANINFO = "conaninfo.txt" CONANENV = "conanenv.txt" SYSTEM_REQS = "system_reqs.txt" -DIRTY_FILE = ".conan_dirty" +SOURCE_DIRTY_FILE = ".conan_source_dirty" PUT_HEADERS = "artifacts.properties" PACKAGE_TGZ_NAME = "conan_package.tgz" @@ -176,6 +176,10 @@ def conan(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) return normpath(join(self._store_folder, "/".join(conan_reference))) + def dirty_sources_file(self, conan_reference): + assert isinstance(conan_reference, ConanFileReference) + return normpath(join(self.conan(conan_reference), SOURCE_DIRTY_FILE)) + def export(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) return normpath(join(self.conan(conan_reference), EXPORT_FOLDER)) diff --git a/conans/test/command/source_test.py b/conans/test/command/source_test.py index 84da89dfe3c..63a42dc1785 100644 --- a/conans/test/command/source_test.py +++ b/conans/test/command/source_test.py @@ -10,12 +10,14 @@ class SourceTest(unittest.TestCase): def basic_source_test(self): conanfile = ''' from conans import ConanFile +import os class ConanLib(ConanFile): name = "Hello" version = "0.1" def source(self): + assert(os.listdir(".") == []) # Not conanfile copied, clean source self.output.info("Running source!") ''' client = TestClient() @@ -60,6 +62,5 @@ def source(self): client.save({CONANFILE: conanfile.replace("err", "")}) client.run("source .") self.assertIn("PROJECT: Configuring sources in", client.user_io.out) - self.assertIn("PROJECT: WARN: Your previous source command failed", client.user_io.out) self.assertIn("PROJECT: Running source!", client.user_io.out) self.assertEqual("Hello World", load(os.path.join(client.current_folder, "file1.txt"))) diff --git a/conans/test/integration/build_id_test.py b/conans/test/integration/build_id_test.py index 8f2536ee063..85153020e61 100644 --- a/conans/test/integration/build_id_test.py +++ b/conans/test/integration/build_id_test.py @@ -1,10 +1,11 @@ -import unittest -from conans.test.utils.tools import TestClient import os -from conans.util.files import load -from conans.model.ref import PackageReference, ConanFileReference +import unittest + from nose_parameterized.parameterized import parameterized +from conans.model.ref import PackageReference, ConanFileReference +from conans.test.utils.tools import TestClient +from conans.util.files import load conanfile = """from conans import ConanFile from conans.util.files import save @@ -56,7 +57,6 @@ def imports(self): class BuildIdTest(unittest.TestCase): - def _check_conaninfo(self, client): # Check that conaninfo is correct ref_debug = PackageReference.loads("Pkg/0.1@user/channel:" diff --git a/conans/test/integration/export_sources_test.py b/conans/test/integration/export_sources_test.py index f90bbdcbd51..537944455cb 100644 --- a/conans/test/integration/export_sources_test.py +++ b/conans/test/integration/export_sources_test.py @@ -85,11 +85,11 @@ def setUp(self): def _check_source_folder(self, mode): """ Source folder MUST be always the same """ - expected_sources = ['conanfile.py', 'conanmanifest.txt', "hello.h"] + expected_sources = ["hello.h"] if mode == "both": expected_sources.append("data.txt") if mode == "nested" or mode == "overlap": - expected_sources = ['conanfile.py', 'conanmanifest.txt', "src/hello.h", "src/data.txt"] + expected_sources = ["src/hello.h", "src/data.txt"] expected_sources = sorted(expected_sources) self.assertEqual(scan_folder(self.source_folder), expected_sources) diff --git a/conans/test/integration/go_complete_test.py b/conans/test/integration/go_complete_test.py index cf61804d86e..8ad3561ef31 100644 --- a/conans/test/integration/go_complete_test.py +++ b/conans/test/integration/go_complete_test.py @@ -92,6 +92,7 @@ def reuse_test(self): 'reverse_test.go': reverse_test, 'reverse.txt': reverse, 'hello/helloreverse.txt': reverse} + files_without_conanfile = set(files.keys()) - set(["conanfile.py"]) self.client.save(files) self.client.run("export lasote/stable") self.client.run("install %s --build missing" % str(conan_reference)) @@ -99,7 +100,7 @@ def reuse_test(self): package_ids = self.client.paths.conan_packages(conan_reference) self.assertEquals(len(package_ids), 1) package_ref = PackageReference(conan_reference, package_ids[0]) - self._assert_package_exists(package_ref, self.client.paths, list(files.keys())) + self._assert_package_exists(package_ref, self.client.paths, files_without_conanfile) # Upload conans self.client.run("upload %s" % str(conan_reference)) @@ -113,7 +114,7 @@ def reuse_test(self): self.client.run("upload %s -p %s" % (str(conan_reference), str(package_ids[0]))) # Check library on server - self._assert_package_exists_in_server(package_ref, server_paths, list(files.keys())) + self._assert_package_exists_in_server(package_ref, server_paths, files_without_conanfile) # Now from other "computer" install the uploaded conans with same options (nothing) other_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) @@ -122,7 +123,7 @@ def reuse_test(self): build_path = other_conan.paths.build(package_ref) self.assertFalse(os.path.exists(build_path)) # Lib should exist - self._assert_package_exists(package_ref, other_conan.paths, list(files.keys())) + self._assert_package_exists(package_ref, other_conan.paths, files_without_conanfile) reuse_conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) files = {'conanfile.py': reuse_conanfile, From 8a488d4de3a73e2e782f47c9f39943767c0c2920 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 11 Sep 2017 16:08:07 +0200 Subject: [PATCH 10/39] Added cwd parameter and fixed current directory for local source method (#1721) --- conans/client/command.py | 3 ++- conans/client/conan_api.py | 1 - conans/client/source.py | 17 +++++++++-------- conans/test/command/source_test.py | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/conans/client/command.py b/conans/client/command.py index 21effb78e05..da3378cbbc0 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -451,9 +451,10 @@ def source(self, *args): " folder, then the execution and retrieval of the source code." " Otherwise, if the code has already been retrieved, it will" " do nothing.") + parser.add_argument("--cwd", "-c", help='Use this directory as the current directory') args = parser.parse_args(*args) - return self._conan.source(args.reference, args.force) + return self._conan.source(args.reference, args.force, cwd=args.cwd) def imports(self, *args): """ Execute the 'imports' stage of a conanfile.txt or a conanfile.py. diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 33e8c8d2806..dbc384e5c07 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -526,7 +526,6 @@ def imports(self, reference, undo=False, dest=None, filename=None, cwd=None): current_path = reference self._manager.imports_undo(current_path) else: - cwd = prepare_cwd(cwd) current_path, reference = _get_reference(reference, cwd) self._manager.imports(current_path, reference, filename, dest) diff --git a/conans/client/source.py b/conans/client/source.py index 6bf2514cc5a..d6c7aa987d1 100644 --- a/conans/client/source.py +++ b/conans/client/source.py @@ -87,11 +87,12 @@ def remove_source(raise_error=True): def config_source_local(current_path, conan_file, output): output.info('Configuring sources in %s' % current_path) - try: - with conanfile_exception_formatter(str(conan_file), "source"): - with tools.environment_append(conan_file.env): - conan_file.source() - except ConanExceptionInUserConanfileMethod: - raise - except Exception as e: - raise ConanException(e) + with tools.chdir(current_path): + try: + with conanfile_exception_formatter(str(conan_file), "source"): + with tools.environment_append(conan_file.env): + conan_file.source() + except ConanExceptionInUserConanfileMethod: + raise + except Exception as e: + raise ConanException(e) diff --git a/conans/test/command/source_test.py b/conans/test/command/source_test.py index 63a42dc1785..2084897a7c1 100644 --- a/conans/test/command/source_test.py +++ b/conans/test/command/source_test.py @@ -38,6 +38,27 @@ def source(self): self.assertIn("Hello/0.1@lasote/stable: Configuring sources", client.user_io.out) self.assertIn("Hello/0.1@lasote/stable: Running source!", client.user_io.out) + def source_local_cwd_test(self): + conanfile = ''' +import os +from conans import ConanFile + +class ConanLib(ConanFile): + name = "Hello" + version = "0.1" + + def source(self): + self.output.info("Running source!") + self.output.info("cwd=>%s" % os.getcwd()) +''' + client = TestClient() + client.save({CONANFILE: conanfile}) + subdir = os.path.join(client.current_folder, "subdir") + os.mkdir(subdir) + client.run("source .. --cwd subdir") + self.assertIn("PROJECT: Configuring sources", client.user_io.out) + self.assertIn("PROJECT: cwd=>%s" % subdir, client.user_io.out) + def local_source_test(self): conanfile = ''' from conans import ConanFile From c7d6e4e7edf920a85a7fc9efbbf5b22e19866bc8 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 11 Sep 2017 18:01:48 +0200 Subject: [PATCH 11/39] Feature/installer refactor (#1741) * Dirty file outside the source folder and not generated by local method * untouched file * Do not copy recipe and manifest to the source folder * Fixed test * installer refactor * pending * No new functionallity, just refactored installer --- conans/client/installer.py | 552 ++++++++++++++++++------------------- conans/client/manager.py | 64 ++++- 2 files changed, 323 insertions(+), 293 deletions(-) diff --git a/conans/client/installer.py b/conans/client/installer.py index e1221537bf2..cddc1b367ae 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -1,7 +1,6 @@ import os import time import platform -import fnmatch import shutil from conans.model.env_info import EnvInfo @@ -10,7 +9,8 @@ from conans.util.files import save, rmdir, mkdir from conans.model.ref import PackageReference from conans.util.log import logger -from conans.errors import ConanException, conanfile_exception_formatter, ConanExceptionInUserConanfileMethod +from conans.errors import (ConanException, conanfile_exception_formatter, + ConanExceptionInUserConanfileMethod) from conans.client.packager import create_package from conans.client.generators import write_generators, TXTGenerator from conans.model.build_info import CppInfo @@ -36,76 +36,225 @@ def _init_package_info(deps_graph, paths, current_path): conan_file.user_info = UserInfo() -def build_id(conanfile): - if hasattr(conanfile, "build_id"): +def build_id(conan_file): + if hasattr(conan_file, "build_id"): # construct new ConanInfo - build_id_info = conanfile.info.copy() - conanfile.info_build = build_id_info + build_id_info = conan_file.info.copy() + conan_file.info_build = build_id_info # effectively call the user function to change the package values - with conanfile_exception_formatter(str(conanfile), "build_id"): - conanfile.build_id() + with conanfile_exception_formatter(str(conan_file), "build_id"): + conan_file.build_id() # compute modified ID return build_id_info.package_id() return None -class BuildMode(object): - def __init__(self, params, output): +class _ConanPackageBuilder(object): + """Builds and packages a single conan_file binary package""" + + def __init__(self, conan_file, package_reference, client_cache, output): + self._client_cache = client_cache + self._conan_file = conan_file self._out = output - self.outdated = False - self.missing = False - self.patterns = [] - self._unused_patterns = [] - self.all = False - if params is None: - return - - assert isinstance(params, list) - if len(params) == 0: - self.all = True + self._package_reference = package_reference + self._conan_ref = self._package_reference.conan + self._build_folder = None + + def build(self): + """Calls the conanfile's build method""" + if not os.path.exists(self.build_folder) or not hasattr(self._conan_file, "build_id"): + # build_id is not caching the build folder, so actually rebuild the package + _handle_system_requirements(self._conan_file, self._package_reference, + self._client_cache, self._out) + with environment_append(self._conan_file.env): + self._build_package() + + def package(self): + """Generate the info txt files and calls the conanfile package method. + Receives que build_folder because it can change if build_id() method exists""" + + # FIXME: Is weak to assign here the recipe_hash + manifest = self._client_cache.load_manifest(self._conan_ref) + self._conan_file.info.recipe_hash = manifest.summary_hash + + # Creating ***info.txt files + save(os.path.join(self.build_folder, CONANINFO), self._conan_file.info.dumps()) + self._out.info("Generated %s" % CONANINFO) + save(os.path.join(self.build_folder, BUILD_INFO), TXTGenerator(self._conan_file).content) + self._out.info("Generated %s" % BUILD_INFO) + + os.chdir(self.build_folder) + + if getattr(self._conan_file, 'no_copy_source', False): + source_folder = self._client_cache.source(self._conan_ref, + self._conan_file.short_paths) else: - never = False - for param in params: - if param == "outdated": - self.outdated = True - elif param == "missing": - self.missing = True - elif param == "never": - never = True - else: - self.patterns.append("%s" % param) - - if never and (self.outdated or self.missing or self.patterns): - raise ConanException("--build=never not compatible with other options") - self._unused_patterns = list(self.patterns) - - def forced(self, reference, conanfile): - if self.all: - return True + source_folder = self.build_folder + with environment_append(self._conan_file.env): + package_folder = self._client_cache.package(self._package_reference, + self._conan_file.short_paths) + create_package(self._conan_file, source_folder, self.build_folder, package_folder, + self._out) + + def _build_package(self): + """ builds the package, creating the corresponding build folder if necessary + and copying there the contents from the src folder. The code is duplicated + in every build, as some configure processes actually change the source + code. Receives the build_folder because it can change if the method build_id() exists + """ - if conanfile.build_policy_always: - out = ScopedOutput(str(reference), self._out) - out.info("Building package from source as defined by build_policy='always'") - return True + package_folder = self._client_cache.package(self._package_reference, + self._conan_file.short_paths) + src_folder = self._client_cache.source(self._conan_ref, self._conan_file.short_paths) + export_folder = self._client_cache.export(self._conan_ref) + export_source_folder = self._client_cache.export_sources(self._conan_ref, + self._conan_file.short_paths) + dirty_file_path = self._client_cache.dirty_sources_file(self._conan_ref) - ref = reference.name - # Patterns to match, if package matches pattern, build is forced - force_build = any([fnmatch.fnmatch(ref, pattern) for pattern in self.patterns]) - return force_build + try: + rmdir(self.build_folder) + rmdir(package_folder) + except Exception as e: + raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n" + "Close any app using it, and retry" % str(e)) - def allowed(self, reference, conanfile): - return (self.missing or self.outdated or self.forced(reference, conanfile) or - conanfile.build_policy_missing) + self._out.info('Building your package in %s' % self.build_folder) + config_source(export_folder, export_source_folder, src_folder, dirty_file_path, + self._conan_file, self._out) + self._out.info('Copying sources to build folder') - def check_matches(self, references): - for pattern in list(self._unused_patterns): - matched = any(fnmatch.fnmatch(ref, pattern) for ref in references) - if matched: - self._unused_patterns.remove(pattern) + def check_max_path_len(src, files): + if platform.system() != "Windows": + return [] + filtered_files = [] + for the_file in files: + source_path = os.path.join(src, the_file) + # Without storage path, just relative + rel_path = os.path.relpath(source_path, src_folder) + dest_path = os.path.normpath(os.path.join(self.build_folder, rel_path)) + # it is NOT that "/" is counted as "\\" so it counts double + # seems a bug in python, overflows paths near the limit of 260, + if len(dest_path) >= 249: + filtered_files.append(the_file) + self._out.warn("Filename too long, file excluded: %s" % dest_path) + return filtered_files + + if getattr(self._conan_file, 'no_copy_source', False): + mkdir(self.build_folder) + self._conan_file.source_folder = src_folder + else: + shutil.copytree(src_folder, self.build_folder, symlinks=True, ignore=check_max_path_len) + logger.debug("Copied to %s" % self.build_folder) + logger.debug("Files copied %s" % os.listdir(self.build_folder)) + self._conan_file.source_folder = self.build_folder + + os.chdir(self.build_folder) + self._conan_file.build_folder = self.build_folder + self._conan_file._conanfile_directory = self.build_folder + # Read generators from conanfile and generate the needed files + logger.debug("Writing generators") + write_generators(self._conan_file, self.build_folder, self._out) + logger.debug("Files copied after generators %s" % os.listdir(self.build_folder)) - def report_matches(self): - for pattern in self._unused_patterns: - self._out.error("No package matching '%s' pattern" % pattern) + # Build step might need DLLs, binaries as protoc to generate source files + # So execute imports() before build, storing the list of copied_files + from conans.client.importer import run_imports + copied_files = run_imports(self._conan_file, self.build_folder, self._out) + + try: + # This is necessary because it is different for user projects + # than for packages + logger.debug("Call conanfile.build() with files in build folder: %s" + % os.listdir(self.build_folder)) + self._out.highlight("Calling build()") + with conanfile_exception_formatter(str(self._conan_file), "build"): + self._conan_file.build() + + self._out.success("Package '%s' built" % self._conan_file.info.package_id()) + self._out.info("Build folder %s" % self.build_folder) + except Exception as exc: + os.chdir(src_folder) + self._out.writeln("") + self._out.error("Package '%s' build failed" % self._conan_file.info.package_id()) + self._out.warn("Build folder %s" % self.build_folder) + if isinstance(exc, ConanExceptionInUserConanfileMethod): + raise exc + raise ConanException(exc) + + finally: + self._conan_file._conanfile_directory = export_folder + # Now remove all files that were imported with imports() + for f in copied_files: + try: + if f.startswith(self.build_folder): + os.remove(f) + except Exception: + self._out.warn("Unable to remove imported file from build: %s" % f) + + @property + def build_folder(self): + if not self._build_folder: + new_id = build_id(self._conan_file) + new_ref = PackageReference(self._conan_ref, new_id) if new_id else self._package_reference + self._build_folder = self._client_cache.build(new_ref, self._conan_file.short_paths) + return self._build_folder + + +def _raise_package_not_found_error(conan_file, conan_ref, out): + settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines()) + options_text = ", ".join(conan_file.info.full_options.dumps().splitlines()) + + out.warn('''Can't find a '%s' package for the specified options and settings: +- Settings: %s +- Options: %s +''' % (conan_ref, settings_text, options_text)) + + raise ConanException('''Missing prebuilt package for '%s' +Try to build it from sources with "--build %s" +Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package" +''' % (conan_ref, conan_ref.name)) + + +def _handle_system_requirements(conan_file, package_reference, client_cache, out): + """ check first the system_reqs/system_requirements.txt existence, if not existing + check package/sha1/ + + Used after remote package retrieving and before package building + """ + if "system_requirements" not in type(conan_file).__dict__: + return + + system_reqs_path = client_cache.system_reqs(package_reference.conan) + system_reqs_package_path = client_cache.system_reqs_package(package_reference) + if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path): + return + + ret = call_system_requirements(conan_file, out) + + try: + ret = str(ret or "") + except: + out.warn("System requirements didn't return a string") + ret = "" + if getattr(conan_file, "global_system_requirements", None): + save(system_reqs_path, ret) + else: + save(system_reqs_package_path, ret) + + +def call_system_requirements(conanfile, output): + try: + return conanfile.system_requirements() + except Exception as e: + output.error("while executing system_requirements(): %s" % str(e)) + raise ConanException("Error in system requirements") + + +def call_package_info(conanfile): + # Once the node is build, execute package info, so it has access to the + # package folder and artifacts + with conanfile_exception_formatter(str(conanfile), "package_info"): + conanfile.package_info() class ConanInstaller(object): @@ -149,7 +298,7 @@ def _compute_private_nodes(self, deps_graph): continue if conan_ref: - build_forced = self._build_mode.forced(conan_ref, conanfile) + build_forced = self._build_mode.forced(conanfile, conan_ref) if build_forced: continue @@ -200,55 +349,79 @@ def _build(self, nodes_by_level, skip_private_nodes, deps_graph): nodes_to_process = self._get_nodes(nodes_by_level, skip_private_nodes) for conan_ref, package_id, conan_file, build_needed in nodes_to_process: + output = ScopedOutput(str(conan_ref), self._out) + package_ref = PackageReference(conan_ref, package_id) + if build_needed and (conan_ref, package_id) not in self._built_packages: - build_allowed = self._build_mode.allowed(conan_ref, conan_file) + build_allowed = self._build_mode.allowed(conan_file, conan_ref) if not build_allowed: - self._raise_package_not_found_error(conan_ref, conan_file) + _raise_package_not_found_error(conan_file, conan_ref, output) - output = ScopedOutput(str(conan_ref), self._out) - package_ref = PackageReference(conan_ref, package_id) - package_folder = self._client_cache.package(package_ref, conan_file.short_paths) if conan_file.build_policy_missing: output.info("Building package from source as defined by build_policy='missing'") - elif self._build_mode.forced(conan_ref, conan_file): + elif self._build_mode.forced(conan_file, conan_ref): output.warn('Forced build from source') self._build_requires.install(conan_ref, conan_file, self) t1 = time.time() # Assign to node the propagated info - self._propagate_info(conan_ref, conan_file, flat, deps_graph) + self._propagate_info(conan_file, conan_ref, flat, deps_graph) self._remote_proxy.get_recipe_sources(conan_ref, conan_file.short_paths) - # Call the conanfile's build method - build_folder = self._build_conanfile(conan_ref, conan_file, package_ref, - package_folder, output) + builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output) + builder.build() + builder.package() - # Call the conanfile's package method - self._package_conanfile(conan_ref, conan_file, package_ref, build_folder, - package_folder, output) + self._remote_proxy.handle_package_manifest(package_ref, installed=True) # Call the info method - self._package_info_conanfile(conan_file) + call_package_info(conan_file) - duration = time.time() - t1 - log_file = os.path.join(build_folder, RUN_LOG_NAME) - log_file = log_file if os.path.exists(log_file) else None - log_package_built(package_ref, duration, log_file) + # Log build + self._log_built_package(conan_file, package_ref, time.time() - t1) self._built_packages.add((conan_ref, package_id)) else: # Get the package, we have a not outdated remote package if conan_ref: - self._get_package(conan_ref, conan_file) + self.get_remote_package(conan_file, package_ref, output) # Assign to the node the propagated info # (conan_ref could be None if user project, but of course assign the info - self._propagate_info(conan_ref, conan_file, flat, deps_graph) + self._propagate_info(conan_file, conan_ref, flat, deps_graph) # Call the info method - self._package_info_conanfile(conan_file) + call_package_info(conan_file) + + def get_remote_package(self, conan_file, package_reference, output): + """Get remote package. It won't check if it's outdated""" + # Compute conan_file package from local (already compiled) or from remote + + package_folder = self._client_cache.package(package_reference, + conan_file.short_paths) + + # If already exists do not dirt the output, the common situation + # is that package is already installed and OK. If don't, the proxy + # will print some other message about it + if not os.path.exists(package_folder): + self._out.info("Retrieving package %s" % package_reference.package_id) + + if self._remote_proxy.get_package(package_reference, + short_paths=conan_file.short_paths): + _handle_system_requirements(conan_file, package_reference, + self._client_cache, output) + return True + + _raise_package_not_found_error(conan_file, package_reference.conan, output) - def _propagate_info(self, conan_ref, conan_file, flat, deps_graph): + def _log_built_package(self, conan_file, package_ref, duration): + build_folder = self._client_cache.build(package_ref, conan_file.short_paths) + log_file = os.path.join(build_folder, RUN_LOG_NAME) + log_file = log_file if os.path.exists(log_file) else None + log_package_built(package_ref, duration, log_file) + + @staticmethod + def _propagate_info(conan_file, conan_ref, flat, deps_graph): # Get deps_cpp_info from upstream nodes node_order = deps_graph.ordered_closure((conan_ref, conan_file), flat) public_deps = [name for name, req in conan_file.requires.items() if not req.private] @@ -266,7 +439,8 @@ def _propagate_info(self, conan_ref, conan_file, flat, deps_graph): subtree_libnames = [ref.name for (ref, _) in node_order] for package_name, env_vars in conan_file._env_values.data.items(): for name, value in env_vars.items(): - if not package_name or package_name in subtree_libnames or package_name == conan_file.name: + if not package_name or package_name in subtree_libnames or \ + package_name == conan_file.name: conan_file.info.env_values.add(name, value, package_name) def _get_nodes(self, nodes_by_level, skip_nodes): @@ -296,12 +470,13 @@ def _get_nodes(self, nodes_by_level, skip_nodes): if package_reference not in package_references: package_references.add(package_reference) check_outdated = self._build_mode.outdated - if self._build_mode.forced(conan_ref, conan_file): + if self._build_mode.forced(conan_file, conan_ref): build_node = True else: - build_node = not self._remote_proxy.package_available(package_reference, - conan_file.short_paths, - check_outdated) + available = self._remote_proxy.package_available(package_reference, + conan_file.short_paths, + check_outdated) + build_node = not available nodes_to_build.append((conan_ref, package_id, conan_file, build_node)) @@ -312,208 +487,3 @@ def _get_nodes(self, nodes_by_level, skip_nodes): self._build_mode.check_matches(to_build) return nodes_to_build - - def _get_package(self, conan_ref, conan_file): - '''Get remote package. It won't check if it's outdated''' - # Compute conan_file package from local (already compiled) or from remote - output = ScopedOutput(str(conan_ref), self._out) - package_id = conan_file.info.package_id() - package_reference = PackageReference(conan_ref, package_id) - - conan_ref = package_reference.conan - package_folder = self._client_cache.package(package_reference, conan_file.short_paths) - - # If already exists do not dirt the output, the common situation - # is that package is already installed and OK. If don't, the proxy - # will print some other message about it - if not os.path.exists(package_folder): - output.info("Retrieving package %s" % package_id) - - if self._remote_proxy.get_package(package_reference, short_paths=conan_file.short_paths): - self._handle_system_requirements(conan_ref, package_reference, conan_file, output) - return True - - self._raise_package_not_found_error(conan_ref, conan_file) - - def _build_conanfile(self, conan_ref, conan_file, package_reference, package_folder, output): - """Calls the conanfile's build method""" - new_id = build_id(conan_file) - if new_id: - package_reference = PackageReference(package_reference.conan, new_id) - build_folder = self._client_cache.build(package_reference, conan_file.short_paths) - if os.path.exists(build_folder) and hasattr(conan_file, "build_id"): - return build_folder - # build_id is not caching the build folder, so actually rebuild the package - src_folder = self._client_cache.source(conan_ref, conan_file.short_paths) - export_folder = self._client_cache.export(conan_ref) - export_source_folder = self._client_cache.export_sources(conan_ref, conan_file.short_paths) - dirty_file_path = self._client_cache.dirty_sources_file(conan_ref) - - self._handle_system_requirements(conan_ref, package_reference, conan_file, output) - with environment_append(conan_file.env): - self._build_package(export_folder, export_source_folder, src_folder, build_folder, - package_folder, dirty_file_path, conan_file, output) - return build_folder - - def _package_conanfile(self, conan_ref, conan_file, package_reference, build_folder, - package_folder, output): - """Generate the info txt files and calls the conanfile package method""" - - # FIXME: Is weak to assign here the recipe_hash - conan_file.info.recipe_hash = self._client_cache.load_manifest(conan_ref).summary_hash - - # Creating ***info.txt files - save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps()) - output.info("Generated %s" % CONANINFO) - save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content) - output.info("Generated %s" % BUILD_INFO) - - os.chdir(build_folder) - - if getattr(conan_file, 'no_copy_source', False): - source_folder = self._client_cache.source(package_reference.conan, - conan_file.short_paths) - else: - source_folder = build_folder - with environment_append(conan_file.env): - create_package(conan_file, source_folder, build_folder, package_folder, output) - self._remote_proxy.handle_package_manifest(package_reference, installed=True) - - def _raise_package_not_found_error(self, conan_ref, conan_file): - settings_text = ", ".join(conan_file.info.full_settings.dumps().splitlines()) - options_text = ", ".join(conan_file.info.full_options.dumps().splitlines()) - - self._out.warn('''Can't find a '%s' package for the specified options and settings: -- Settings: %s -- Options: %s -''' % (conan_ref, settings_text, options_text)) - - raise ConanException('''Missing prebuilt package for '%s' -Try to build it from sources with "--build %s" -Or read "http://docs.conan.io/en/latest/faq/troubleshooting.html#error-missing-prebuilt-package" -''' % (conan_ref, conan_ref.name)) - - def _handle_system_requirements(self, conan_ref, package_reference, conan_file, coutput): - """ check first the system_reqs/system_requirements.txt existence, if not existing - check package/sha1/ - """ - if "system_requirements" not in type(conan_file).__dict__: - return - - system_reqs_path = self._client_cache.system_reqs(conan_ref) - system_reqs_package_path = self._client_cache.system_reqs_package(package_reference) - if os.path.exists(system_reqs_path) or os.path.exists(system_reqs_package_path): - return - - output = self.call_system_requirements(conan_file, coutput) - - try: - output = str(output or "") - except: - coutput.warn("System requirements didn't return a string") - output = "" - if getattr(conan_file, "global_system_requirements", None): - save(system_reqs_path, output) - else: - save(system_reqs_package_path, output) - - def call_system_requirements(self, conan_file, output): - try: - return conan_file.system_requirements() - except Exception as e: - output.error("while executing system_requirements(): %s" % str(e)) - raise ConanException("Error in system requirements") - - def _build_package(self, export_folder, export_source_folder, src_folder, build_folder, - package_folder, dirty_file_path, conan_file, output): - """ builds the package, creating the corresponding build folder if necessary - and copying there the contents from the src folder. The code is duplicated - in every build, as some configure processes actually change the source - code - """ - - try: - rmdir(build_folder) - rmdir(package_folder) - except Exception as e: - raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n" - "Close any app using it, and retry" % str(e)) - - output.info('Building your package in %s' % build_folder) - config_source(export_folder, export_source_folder, src_folder, dirty_file_path, - conan_file, output) - output.info('Copying sources to build folder') - - def check_max_path_len(src, files): - if platform.system() != "Windows": - return [] - filtered_files = [] - for the_file in files: - source_path = os.path.join(src, the_file) - # Without storage path, just relative - rel_path = os.path.relpath(source_path, src_folder) - dest_path = os.path.normpath(os.path.join(build_folder, rel_path)) - # it is NOT that "/" is counted as "\\" so it counts double - # seems a bug in python, overflows paths near the limit of 260, - if len(dest_path) >= 249: - filtered_files.append(the_file) - output.warn("Filename too long, file excluded: %s" % dest_path) - return filtered_files - - if getattr(conan_file, 'no_copy_source', False): - mkdir(build_folder) - conan_file.source_folder = src_folder - else: - shutil.copytree(src_folder, build_folder, symlinks=True, ignore=check_max_path_len) - logger.debug("Copied to %s" % build_folder) - logger.debug("Files copied %s" % os.listdir(build_folder)) - conan_file.source_folder = build_folder - - os.chdir(build_folder) - conan_file.build_folder = build_folder - conan_file._conanfile_directory = build_folder - # Read generators from conanfile and generate the needed files - logger.debug("Writing generators") - write_generators(conan_file, build_folder, output) - logger.debug("Files copied after generators %s" % os.listdir(build_folder)) - - # Build step might need DLLs, binaries as protoc to generate source files - # So execute imports() before build, storing the list of copied_files - from conans.client.importer import run_imports - copied_files = run_imports(conan_file, build_folder, output) - - try: - # This is necessary because it is different for user projects - # than for packages - logger.debug("Call conanfile.build() with files in build folder: %s" - % os.listdir(build_folder)) - output.highlight("Calling build()") - with conanfile_exception_formatter(str(conan_file), "build"): - conan_file.build() - - output.success("Package '%s' built" % conan_file.info.package_id()) - output.info("Build folder %s" % build_folder) - except Exception as exc: - os.chdir(src_folder) - self._out.writeln("") - output.error("Package '%s' build failed" % conan_file.info.package_id()) - output.warn("Build folder %s" % build_folder) - if isinstance(exc, ConanExceptionInUserConanfileMethod): - raise exc - raise ConanException(exc) - - finally: - conan_file._conanfile_directory = export_folder - # Now remove all files that were imported with imports() - for f in copied_files: - try: - if(f.startswith(build_folder)): - os.remove(f) - except Exception: - self._out.warn("Unable to remove imported file from build: %s" % f) - - def _package_info_conanfile(self, conan_file): - # Once the node is build, execute package info, so it has access to the - # package folder and artifacts - with conanfile_exception_formatter(str(conan_file), "package_info"): - conan_file.package_info() diff --git a/conans/client/manager.py b/conans/client/manager.py index a0edecfb98a..1af1d49dd73 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -1,3 +1,4 @@ +import fnmatch import os import time import shutil @@ -13,7 +14,7 @@ from conans.client.generators import write_generators from conans.client.generators.text import TXTGenerator from conans.client.importer import run_imports, undo_imports -from conans.client.installer import ConanInstaller, BuildMode +from conans.client.installer import ConanInstaller, call_system_requirements from conans.client.loader import ConanFileLoader from conans.client.manifest_manager import ManifestManager from conans.client.output import ScopedOutput, Color @@ -40,6 +41,65 @@ from conans.search.search import filter_outdated +class BuildMode(object): + def __init__(self, params, output): + self._out = output + self.outdated = False + self.missing = False + self.patterns = [] + self._unused_patterns = [] + self.all = False + if params is None: + return + + assert isinstance(params, list) + if len(params) == 0: + self.all = True + else: + never = False + for param in params: + if param == "outdated": + self.outdated = True + elif param == "missing": + self.missing = True + elif param == "never": + never = True + else: + self.patterns.append("%s" % param) + + if never and (self.outdated or self.missing or self.patterns): + raise ConanException("--build=never not compatible with other options") + self._unused_patterns = list(self.patterns) + + def forced(self, conan_file, reference): + if self.all: + return True + + if conan_file.build_policy_always: + out = ScopedOutput(str(reference), self._out) + out.info("Building package from source as defined by build_policy='always'") + return True + + ref = reference.name + # Patterns to match, if package matches pattern, build is forced + force_build = any([fnmatch.fnmatch(ref, pattern) for pattern in self.patterns]) + return force_build + + def allowed(self, conan_file, reference): + return (self.missing or self.outdated or self.forced(conan_file, reference) or + conan_file.build_policy_missing) + + def check_matches(self, references): + for pattern in list(self._unused_patterns): + matched = any(fnmatch.fnmatch(ref, pattern) for ref in references) + if matched: + self._unused_patterns.remove(pattern) + + def report_matches(self): + for pattern in self._unused_patterns: + self._out.error("No package matching '%s' pattern" % pattern) + + class ConanManager(object): """ Manage all the commands logic The main entry point for all the client business logic @@ -359,7 +419,7 @@ def install(self, reference, current_path, profile, remote=None, output.info("Generated %s" % CONANINFO) if not no_imports: run_imports(conanfile, current_path, output) - installer.call_system_requirements(conanfile, output) + call_system_requirements(conanfile, output) if manifest_manager: manifest_manager.print_log() From 24643cce204cde71c8c7b22927b2bb8e561367f1 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 11 Sep 2017 22:17:51 +0200 Subject: [PATCH 12/39] Feature/full traceback (#1703) * Progress not fully valid, missing external module dection and classic format * Review --- conans/client/conf/__init__.py | 5 ++- conans/errors.py | 36 +++++++++++++------ .../integration/exception_printing_test.py | 29 +++++++++------ 3 files changed, 49 insertions(+), 21 deletions(-) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index fe843c52e73..1f87bc1118e 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -5,7 +5,7 @@ from conans.errors import ConanException from conans.model.env_info import unquote -from conans.paths import conan_expand_user, DEFAULT_PROFILE_NAME, get_conan_user_home +from conans.paths import conan_expand_user, DEFAULT_PROFILE_NAME from conans.util.env_reader import get_env from conans.util.files import load @@ -60,10 +60,12 @@ default_profile = %s compression_level = 9 # environment CONAN_COMPRESSION_LEVEL sysrequires_sudo = True # environment CONAN_SYSREQUIRES_SUDO +# verbose_traceback = False # environment CONAN_VERBOSE_TRACEBACK # bash_path = "" # environment CONAN_BASH_PATH (only windows) # recipe_linter = False # environment CONAN_RECIPE_LINTER # pylintrc = path/to/pylintrc_file # environment CONAN_PYLINTRC + # cmake_generator # environment CONAN_CMAKE_GENERATOR # http://www.vtk.org/Wiki/CMake_Cross_Compiling # cmake_toolchain_file # environment CONAN_CMAKE_TOOLCHAIN_FILE @@ -122,6 +124,7 @@ def env_vars(self): "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_RECIPE_LINTER": self._env_c("general.recipe_linter", "CONAN_RECIPE_LINTER", "True"), "CONAN_CPU_COUNT": self._env_c("general.cpu_count", "CONAN_CPU_COUNT", None), + "CONAN_VERBOSE_TRACEBACK": self._env_c("general.verbose_traceback", "CONAN_VERBOSE_TRACEBACK", None), # http://www.vtk.org/Wiki/CMake_Cross_Compiling "CONAN_CMAKE_GENERATOR": self._env_c("general.cmake_generator", "CONAN_CMAKE_GENERATOR", None), "CONAN_CMAKE_TOOLCHAIN_FILE": self._env_c("general.cmake_toolchain_file", "CONAN_CMAKE_TOOLCHAIN_FILE", None), diff --git a/conans/errors.py b/conans/errors.py index 70cf7327d05..0fcbadff53e 100644 --- a/conans/errors.py +++ b/conans/errors.py @@ -8,9 +8,10 @@ see return_plugin.py """ - from contextlib import contextmanager +from conans.util.env_reader import get_env + @contextmanager def conanfile_exception_formatter(conanfile_name, func_name): @@ -27,25 +28,40 @@ def conanfile_exception_formatter(conanfile_name, func_name): def _format_conanfile_exception(scope, method, exception): + """ + It will iterate the traceback lines, when it finds that the source code is inside the users + conanfile it "start recording" the messages, when the trace exits the conanfile we return + the traces. + """ import sys import traceback - msg = "%s: Error in %s() method" % (scope, method) + if get_env("CONAN_VERBOSE_TRACEBACK", False): + return traceback.format_exc() try: + conanfile_reached = False tb = sys.exc_info()[2] - index = -1 + index = 0 + content_lines = [] + while True: # If out of index will raise and will be captured later filepath, line, name, contents = traceback.extract_tb(tb, 40)[index] # 40 levels of nested functions max, get the latest if "conanfile.py" not in filepath: # Avoid show trace from internal conan source code - index -= 1 + if conanfile_reached: # The error goes to internal code, exit print + break else: - break - if name != method: - msg += ", while calling '%s'" % name - msg += ", line %d\n\t%s" % (line, contents) if line else "\n\t%s" % contents + if not conanfile_reached: # First line + msg = "%s: Error in %s() method" % (scope, method) + msg += ", line %d\n\t%s" % (line, contents) + else: + msg = "while calling '%s', line %d\n\t%s" % (name, line, contents) if line else "\n\t%s" % contents + content_lines.append(msg) + conanfile_reached = True + index += 1 except: pass - msg += "\n\t%s: %s" % (exception.__class__.__name__, str(exception)) - return msg + ret = "\n".join(content_lines) + ret += "\n\t%s: %s" % (exception.__class__.__name__, str(exception)) + return ret class ConanException(Exception): diff --git a/conans/test/integration/exception_printing_test.py b/conans/test/integration/exception_printing_test.py index 3c3ea4f9c40..62ad9bd7433 100644 --- a/conans/test/integration/exception_printing_test.py +++ b/conans/test/integration/exception_printing_test.py @@ -52,7 +52,6 @@ def _aux_method(self): def setUp(self): self.client = TestClient() - def _call_install(self, conanfile): self.client.save({CONANFILE: conanfile}, clean_first=True) self.client.run("export lasote/stable") @@ -78,10 +77,13 @@ def _get_conanfile_for(self, method_name): config_options_contents=throw if method_name == "config_options" else "pass") return cf - def _test_fail_line_aux(self, conanfile, numline, method_name): + def _test_fail_line_aux(self, conanfile, main_line, numline, method_name): self._call_install(conanfile) - self.assertIn("ExceptionsTest/0.1@lasote/stable: Error in %s() method, while calling '_aux_method', line %s" % (method_name, numline), + self.assertIn("ExceptionsTest/0.1@lasote/stable: Error in %s() method, line %s" % (method_name, main_line), + self.client.user_io.out) + self.assertIn("\nwhile calling '_aux_method', line %s" % numline, self.client.user_io.out) + self.assertIn("DRLException: Oh! an error!", self.client.user_io.out) def _get_conanfile_for_error_in_other_method(self, method_name): @@ -107,13 +109,20 @@ def test_all_methods(self): self._test_fail_line(self._get_conanfile_for(method), line, method) def test_aux_method(self): - for method, line in [("source", 41), ("build", 41), - ("package", 41), ("package_info", 41), - ("configure", 41), ("build_id", 41), - ("package_id", 41), ("requirements", 41), - ("config_options", 41)]: - self._test_fail_line_aux(self._get_conanfile_for_error_in_other_method(method), line, method) - + for method, main_line, line in [("source", 14, 41), ("build", 17, 41), + ("package", 20, 41), ("package_info", 23, 41), + ("configure", 26, 41), ("build_id", 29, 41), + ("package_id", 32, 41), ("requirements", 35, 41), + ("config_options", 38, 41)]: + self._test_fail_line_aux(self._get_conanfile_for_error_in_other_method(method), + main_line, line, method) + + def test_complete_traceback(self): + with tools.environment_append({"CONAN_VERBOSE_TRACEBACK": "1"}): + self._call_install(self._get_conanfile_for_error_in_other_method("source")) + self.assertIn("ERROR: Traceback (most recent call last):", self.client.user_io.out) + self.assertIn('self._aux_method()', self.client.user_io.out) + self.assertIn("raise DRLException('Oh! an error!')", self.client.user_io.out) if __name__ == '__main__': From 8442e7c0c9e7fb23a8e6f2e2dd6525dc2024f89a Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 09:12:40 +0200 Subject: [PATCH 13/39] Feature/improve tests vs12 (#1739) * added nosetests mingw attr, and remove testing dep to VS12 * fixed broken test in linux --- conans/test/command/install_subfolder_test.py | 6 +- .../test/functional/compile_helpers_test.py | 2 +- .../integration/build_environment_test.py | 3 + conans/test/integration/cmake_multi_test.py | 1 + conans/test/libcxx_setting_test.py | 109 ++++++++---------- 5 files changed, 56 insertions(+), 65 deletions(-) diff --git a/conans/test/command/install_subfolder_test.py b/conans/test/command/install_subfolder_test.py index 1a953aebb8a..e4145586439 100644 --- a/conans/test/command/install_subfolder_test.py +++ b/conans/test/command/install_subfolder_test.py @@ -66,6 +66,6 @@ def reuse_test(self): (1, h01, h11, h00, h10)]: self.client.current_folder = os.path.join(current_folder, "lang%dbuild" % lang) self.client.run("build ..") - self.assertIn("compiler=Visual Studio", self.client.user_io.out) - self.assertIn("language=%d" % lang, self.client.user_io.out) - self.assertNotIn("language=%d" % (not lang), self.client.user_io.out) + self.assertIn("compiler=Visual Studio", self.client.out) + self.assertIn("language=%d" % lang, self.client.out) + self.assertNotIn("language=%d" % (not lang), self.client.out) diff --git a/conans/test/functional/compile_helpers_test.py b/conans/test/functional/compile_helpers_test.py index 37876e9aed2..fc93ee8656c 100644 --- a/conans/test/functional/compile_helpers_test.py +++ b/conans/test/functional/compile_helpers_test.py @@ -419,7 +419,7 @@ def append_variables_test(self): win_settings = MockSettings("Release", os="Windows", arch="x86", compiler_name="Visual Studio", libcxx=None, - version="12") + version="14") env = ConfigureEnvironment(MockConanfile(win_settings)) command = "%s && SET" % env.command_line runner(command, output=output) diff --git a/conans/test/integration/build_environment_test.py b/conans/test/integration/build_environment_test.py index b108d8a7647..eeae42e1fc9 100644 --- a/conans/test/integration/build_environment_test.py +++ b/conans/test/integration/build_environment_test.py @@ -2,6 +2,8 @@ import platform import unittest +from nose.plugins.attrib import attr + from conans.model.ref import ConanFileReference from conans.paths import CONANFILE from conans.test.utils.tools import TestClient @@ -63,6 +65,7 @@ def package_info(self): class BuildEnvironmenTest(unittest.TestCase): + @attr("mingw") def test_gcc_and_environment(self): if platform.system() == "SunOS": return # If is using sun-cc the gcc generator doesn't work diff --git a/conans/test/integration/cmake_multi_test.py b/conans/test/integration/cmake_multi_test.py index 152c4d60585..a85668f4660 100644 --- a/conans/test/integration/cmake_multi_test.py +++ b/conans/test/integration/cmake_multi_test.py @@ -130,6 +130,7 @@ def package_files(name, deps=None): @attr("slow") class CMakeMultiTest(unittest.TestCase): + @attr("mingw") def cmake_multi_find_test(self): if platform.system() not in ["Windows", "Linux"]: return diff --git a/conans/test/libcxx_setting_test.py b/conans/test/libcxx_setting_test.py index 2d5247a8f71..4c46f3e9853 100644 --- a/conans/test/libcxx_setting_test.py +++ b/conans/test/libcxx_setting_test.py @@ -12,8 +12,6 @@ class ConanFileToolsTest(ConanFile): name = "test" version = "1.9" settings = "os", "compiler", "arch", "build_type" - url = "1" - license = "2" export = ["CMakeLists.txt", "main.c"] generators = ["cmake"] @@ -21,14 +19,13 @@ def build(self): self.output.warn("Building...") cmake = CMake(self) self.output.warn(cmake.command_line) - command = cmake.command_line.replace('-G "Visual Studio 12 Win64"', "") - self.run('cmake . %s' % command) + self.run('cmake . %s' % cmake.command_line) self.run("cmake --build . %s" % cmake.build_config) def package(self): - self.copy("*", ".", ".") - + self.copy("*") ''' + cmakelists = '''PROJECT(conanzlib) set(CONAN_DISABLE_CHECK_COMPILER TRUE) cmake_minimum_required(VERSION 2.8) @@ -50,95 +47,85 @@ def nowintest(func): class LibcxxSettingTest(unittest.TestCase): - def setUp(self): - self.files = {"conanfile.py": file_content, "CMakeLists.txt": cmakelists} - @nowintest def test_declared_stdlib_and_passed(self): client = TestClient() - client.save(self.files) + client.save({"conanfile.py": file_content, + "CMakeLists.txt": cmakelists}) client.run("export lasote/testing") if platform.system() == "SunOS": - client.run('install -s compiler=sun-cc -s compiler.libcxx=libCstd', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libCstd') client.run('build') - self.assertIn("-library=Cstd", str(client.user_io.out)) + self.assertIn("-library=Cstd", client.out) - client.run('install -s compiler=sun-cc -s compiler.libcxx=libstdcxx', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libstdcxx') client.run('build') - self.assertIn("-library=stdcxx4", str(client.user_io.out)) + self.assertIn("-library=stdcxx4", client.out) - client.run('install -s compiler=sun-cc -s compiler.libcxx=libstlport', ignore_error=False) + client.run('install -s compiler=sun-cc -s compiler.libcxx=libstlport') client.run('build') - self.assertIn("-library=stlport4", str(client.user_io.out)) + self.assertIn("-library=stlport4", client.out) else: - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++ ', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++ ') client.run('build') - self.assertIn("-stdlib=libstdc++", str(client.user_io.out)) - self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=0", str(client.user_io.out)) + self.assertIn("-stdlib=libstdc++", client.out) + self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=0", client.out) - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++11', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libstdc++11') client.run('build') - self.assertIn("-stdlib=libstdc++", str(client.user_io.out)) - self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=1", str(client.user_io.out)) + self.assertIn("-stdlib=libstdc++", client.out) + self.assertIn("Found Define: _GLIBCXX_USE_CXX11_ABI=1", client.out) - client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libc++', ignore_error=False) + client.run('install -s compiler=clang -s compiler.version=3.3 -s compiler.libcxx=libc++') client.run('build') - self.assertIn("-stdlib=libc++", str(client.user_io.out)) - self.assertNotIn("Found Define: _GLIBCXX_USE_CXX11", str(client.user_io.out)) + self.assertIn("-stdlib=libc++", client.out) + self.assertNotIn("Found Define: _GLIBCXX_USE_CXX11", client.out) def test_C_only(self): - config = ''' - def config(self): + conanfile = """from conans import ConanFile + +class ConanFileToolsTest(ConanFile): + name = "test" + version = "1.9" + settings = "os", "compiler", "arch", "build_type" + + def configure(self): del self.settings.compiler.libcxx # C package only -''' - self.files["conanfile.py"] = self.files["conanfile.py"].replace('["cmake"]', - '["cmake"]\n %s' % config) + """ - self.files["conanfile.py"] = self.files["conanfile.py"].replace("def build", "def nobuild") client = TestClient() - client.save(self.files) - client.run("export lasote/testing") - client.run("install") + client.save({"conanfile.py": conanfile}) # Also check that it not fails the config method with Visual Studio, because of the lack of libcxx - client.run('install -s compiler="Visual Studio" -s compiler.version=12 -s compiler.runtime=MD', ignore_error=False) - self.assertIn("Generator cmake created conanbuildinfo.cmake", str(client.user_io.out)) + client.run('install -s compiler="Visual Studio" -s compiler.version=14') + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) - self.assertNotIn("libcxx", conaninfo[:conaninfo.find("[full_settings]")]) - client.run('install test/1.9@lasote/testing -s compiler=gcc -s compiler.version=4.9 --build', ignore_error=False) + self.assertNotIn("libcxx", conaninfo) + client.run('install -s compiler=gcc -s compiler.version=4.9') + conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) + self.assertNotIn("libcxx", conaninfo) + + client.run("create lasote/testing -s compiler=gcc -s compiler.version=4.9") # Now try to reuse the installed package defining libstc++11 for the new package newlib_content = ''' from conans import ConanFile, CMake class ConanFileToolsTest(ConanFile): - name = "test2" - version = "1.9" settings = "os", "compiler", "arch", "build_type" - url = "1" - license = "2" - export = ["CMakeLists.txt", "main.c"] - generators = ["cmake"] requires = "test/1.9@lasote/testing" - - def build(self): - pass ''' - new_client = TestClient(base_folder=client.base_folder) # Share storage - new_client.save({"conanfile.py": newlib_content, "CMakeLists.txt": cmakelists}) - new_client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9', ignore_error=False) - # Package is found and everything is ok - self.assertIn("Generator cmake created conanbuildinfo.cmake", str(new_client.user_io.out)) - # Try again without removing the setting, if we use libstdc++11, the C package won't be found - self.files["conanfile.py"] = self.files["conanfile.py"].replace("def config", "def config222") - client.save(self.files) - client.run("export lasote/testing") - client.run("install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9") + client.save({"conanfile.py": newlib_content}) + client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9') + # Package is found and everything is ok + self.assertIn("test/1.9@lasote/testing: Already installed!", client.out) + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) conaninfo = load(os.path.join(client.current_folder, "conaninfo.txt")) - self.assertIn("libcxx", conaninfo[:conaninfo.find("[full_settings]")]) - client.run('install test/1.9@lasote/testing -s compiler=gcc --build -s compiler.libcxx=libstdc++ -s compiler.version=4.9', ignore_error=False) - new_client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++11 -s compiler.version=4.9', ignore_error=True) - self.assertIn("Can't find a 'test/1.9@lasote/testing' package for the specified options and settings", str(new_client.user_io.out)) + self.assertIn("libcxx", conaninfo) + client.run('install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9') + # Package is found and everything is ok + self.assertIn("test/1.9@lasote/testing: Already installed!", client.out) + self.assertIn("PROJECT: Generated conaninfo.txt", client.out) From b54ad130f8c97d49cf82816307478ec56e7a7ac7 Mon Sep 17 00:00:00 2001 From: Tomislav Ivek Date: Tue, 12 Sep 2017 09:13:48 +0200 Subject: [PATCH 14/39] allow pylint versions 1.6.5 up to 1.8.0 (#1724) --- conans/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/requirements.txt b/conans/requirements.txt index 9f6676a975c..62427ce9c52 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -7,7 +7,7 @@ fasteners>=0.14.1 six>=1.10.0 node-semver==0.1.1 distro>=1.0.2, <1.1.0 -pylint==1.6.5 +pylint>=1.6.5, <=1.8.0 future==0.16.0 pygments>=2.0, <3.0 From b33c159fafab86b09c223ce0fb6982984e81d708 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 10:08:20 +0200 Subject: [PATCH 15/39] fixed regex for deps_cpp_info (#1742) --- conans/client/generators/text.py | 2 +- conans/test/command/build_test.py | 29 ++++++++++++++++++++++++++++ conans/test/model/build_info_test.py | 4 +++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/conans/client/generators/text.py b/conans/client/generators/text.py index 96dc5025475..3e549b9d0dc 100644 --- a/conans/client/generators/text.py +++ b/conans/client/generators/text.py @@ -65,7 +65,7 @@ def _loads_deps_user_info(text): @staticmethod def _loads_cpp_info(text): - pattern = re.compile(r"^\[([a-zA-Z0-9_:-\\.]+)\]([^\[]+)", re.MULTILINE) + pattern = re.compile(r"^\[([a-zA-Z0-9._:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: diff --git a/conans/test/command/build_test.py b/conans/test/command/build_test.py index e0da0f1b0bf..a7f90a37b10 100644 --- a/conans/test/command/build_test.py +++ b/conans/test/command/build_test.py @@ -85,6 +85,35 @@ def build_test(self): self.assertIn("Project: HELLO INCLUDE PATHS: %s/include" % package_folder, client.user_io.out) + def build_dots_names_test(self): + """ Try to reuse variables loaded from txt generator => deps_cpp_info + """ + client = TestClient() + conanfile_dep = """ +from conans import ConanFile + +class AConan(ConanFile): + pass +""" + client.save({CONANFILE: conanfile_dep}) + client.run("create Hello.Pkg/0.1@lasote/testing") + client.run("create Hello-Tools/0.1@lasote/testing") + conanfile_scope_env = """ +from conans import ConanFile + +class AConan(ConanFile): + requires = "Hello.Pkg/0.1@lasote/testing", "Hello-Tools/0.1@lasote/testing" + + def build(self): + self.output.info("HELLO ROOT PATH: %s" % self.deps_cpp_info["Hello.Pkg"].rootpath) + self.output.info("HELLO ROOT PATH: %s" % self.deps_cpp_info["Hello-Tools"].rootpath) +""" + client.save({CONANFILE: conanfile_scope_env}, clean_first=True) + client.run("install --build=missing -g txt") + client.run("build") + self.assertIn("Hello.Pkg/0.1/lasote/testing", client.out) + self.assertIn("Hello-Tools/0.1/lasote/testing", client.out) + def build_cmake_install_test(self): client = TestClient() conanfile = """ diff --git a/conans/test/model/build_info_test.py b/conans/test/model/build_info_test.py index 15e0a8e3f26..83694726dfe 100644 --- a/conans/test/model/build_info_test.py +++ b/conans/test/model/build_info_test.py @@ -22,13 +22,15 @@ def parse_test(self): otherlib_path [includedirs_My.Component.Lib] my_component_lib +[includedirs_My-Component-Tool] +my-component-tool """ deps_info, _ = TXTGenerator.loads(text) self.assertEqual(deps_info.includedirs, ['C:/Whenever']) self.assertEqual(deps_info["Boost"].includedirs, ['F:/ChildrenPath']) self.assertEqual(deps_info["My_Lib"].includedirs, ['mylib_path']) self.assertEqual(deps_info["My_Other_Lib"].includedirs, ['otherlib_path']) - self.assertEqual(deps_info["My.Component.Lib"].includedirs, ['my_component_lib']) + self.assertEqual(deps_info["My-Component-Tool"].includedirs, ['my-component-tool']) def help_test(self): deps_env_info = DepsEnvInfo() From 42552fc54f1f7791f6f9ac6e51b7ae1e89cd7507 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 10:08:55 +0200 Subject: [PATCH 16/39] operator 'in' for settings values (#1734) --- conans/model/settings.py | 3 +++ conans/test/model/settings_test.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/conans/model/settings.py b/conans/model/settings.py index ffc3ab707da..38f83fe7b82 100644 --- a/conans/model/settings.py +++ b/conans/model/settings.py @@ -40,6 +40,9 @@ def __init__(self, definition, name): # list or tuple of possible values self._definition = sorted(str(v) for v in definition) + def __contains__(self, value): + return value in (self._value or "") + def copy(self): """ deepcopy, recursive """ diff --git a/conans/test/model/settings_test.py b/conans/test/model/settings_test.py index 399c91a93b4..4a962f649bb 100644 --- a/conans/test/model/settings_test.py +++ b/conans/test/model/settings_test.py @@ -18,6 +18,11 @@ def setUp(self): "os": ["Windows", "Linux"]} self.sut = Settings(data) + def test_in_contains(self): + self.sut.compiler = "Visual Studio" + self.assertTrue("Visual" in self.sut.compiler) + self.assertFalse("Visual" not in self.sut.compiler) + def test_os_split(self): settings = Settings.loads("""os: Windows: From 7c40b85b8dea8410af41ec7f613e6f1c065be5b1 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 12:58:57 +0200 Subject: [PATCH 17/39] Visual studio legacy (#1727) * Implement vsprops creation. * Add a simple testcase for the nbew visual_studio_legacy generator. * minor style and some boy-scout rule * merged develop and fixed test * updated contributors * added contributor --- conans/client/generators/__init__.py | 3 ++ .../client/generators/visualstudiolegacy.py | 40 +++++++++++++++++++ conans/model/conan_generator.py | 4 +- conans/test/generators/generators_test.py | 4 +- .../generators/visual_studio_legacy_test.py | 40 +++++++++++++++++++ conans/test/generators/visual_studio_test.py | 18 ++------- contributors.txt | 1 + 7 files changed, 93 insertions(+), 17 deletions(-) create mode 100644 conans/client/generators/visualstudiolegacy.py create mode 100644 conans/test/generators/visual_studio_legacy_test.py diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index c4306b5c568..1abcabb296e 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -10,6 +10,7 @@ from .qbs import QbsGenerator from .scons import SConsGenerator from .visualstudio import VisualStudioGenerator +from .visualstudiolegacy import VisualStudioLegacyGenerator from .xcode import XCodeGenerator from .ycm import YouCompleteMeGenerator from .virtualenv import VirtualEnvGenerator @@ -22,6 +23,7 @@ def _save_generator(name, klass): if name not in registered_generators: registered_generators.add(name, klass) + _save_generator("txt", TXTGenerator) _save_generator("gcc", GCCGenerator) _save_generator("cmake", CMakeGenerator) @@ -30,6 +32,7 @@ def _save_generator(name, klass): _save_generator("qbs", QbsGenerator) _save_generator("scons", SConsGenerator) _save_generator("visual_studio", VisualStudioGenerator) +_save_generator("visual_studio_legacy", VisualStudioLegacyGenerator) _save_generator("xcode", XCodeGenerator) _save_generator("ycm", YouCompleteMeGenerator) _save_generator("virtualenv", VirtualEnvGenerator) diff --git a/conans/client/generators/visualstudiolegacy.py b/conans/client/generators/visualstudiolegacy.py new file mode 100644 index 00000000000..77e2539e1b7 --- /dev/null +++ b/conans/client/generators/visualstudiolegacy.py @@ -0,0 +1,40 @@ +from conans.model import Generator + + +class VisualStudioLegacyGenerator(Generator): + template = ''' + + + +''' + + @property + def filename(self): + return 'conanbuildinfo.vsprops' + + @property + def content(self): + fields = { + 'include_dirs': "".join(""%s";" % p for p in self._deps_build_info.include_paths).replace("\\", "/"), + 'lib_dirs': "".join(""%s";" % p for p in self._deps_build_info.lib_paths).replace("\\", "/"), + 'libs': "".join(['%s.lib ' % lib if not lib.endswith(".lib") + else '%s ' % lib for lib in self._deps_build_info.libs]), + 'definitions': "".join("%s;" % d for d in self._deps_build_info.defines), + 'compiler_flags': " ".join(self._deps_build_info.cppflags + self._deps_build_info.cflags), + 'linker_flags': " ".join(self._deps_build_info.sharedlinkflags), + } + return self.template.format(**fields) diff --git a/conans/model/conan_generator.py b/conans/model/conan_generator.py index f81a236d5c3..26ec978610f 100644 --- a/conans/model/conan_generator.py +++ b/conans/model/conan_generator.py @@ -24,11 +24,11 @@ def build_info(self): @property def deps_env_info(self): return self._deps_env_info - + @property def deps_user_info(self): return self._deps_user_info - + @property def env_info(self): return self._env_info diff --git a/conans/test/generators/generators_test.py b/conans/test/generators/generators_test.py index abf36a6d1ac..eab8e62561c 100644 --- a/conans/test/generators/generators_test.py +++ b/conans/test/generators/generators_test.py @@ -16,6 +16,7 @@ def test_base(self): scons txt visual_studio +visual_studio_legacy xcode ycm ''' @@ -26,7 +27,8 @@ def test_base(self): self.assertEqual(sorted(['conanfile.txt', 'conaninfo.txt', 'conanbuildinfo.cmake', 'conanbuildinfo.gcc', 'conanbuildinfo.qbs', 'conanbuildinfo.pri', 'SConscript_conan', 'conanbuildinfo.txt', 'conanbuildinfo.props', - 'conanbuildinfo.xcconfig', '.ycm_extra_conf.py']), + 'conanbuildinfo.vsprops', 'conanbuildinfo.xcconfig', + '.ycm_extra_conf.py']), sorted(os.listdir(client.current_folder))) def test_qmake(self): diff --git a/conans/test/generators/visual_studio_legacy_test.py b/conans/test/generators/visual_studio_legacy_test.py new file mode 100644 index 00000000000..c12dc7e0c0a --- /dev/null +++ b/conans/test/generators/visual_studio_legacy_test.py @@ -0,0 +1,40 @@ +import unittest +import xml.etree.ElementTree + +from conans.client.generators import VisualStudioLegacyGenerator + +from conans.model.settings import Settings +from conans.model.conan_file import ConanFile +from conans.model.build_info import CppInfo +from conans.model.ref import ConanFileReference +from conans.test.utils.test_files import temp_folder +import os + + +class VisualStudioLegacyGeneratorTest(unittest.TestCase): + + def valid_xml_test(self): + conanfile = ConanFile(None, None, Settings({}), None) + ref = ConanFileReference.loads("MyPkg/0.1@user/testing") + folder1 = temp_folder() + folder1 = folder1.replace("\\", "/") + os.makedirs(os.path.join(folder1, "include")) + os.makedirs(os.path.join(folder1, "lib")) + cpp_info = CppInfo(folder1) + conanfile.deps_cpp_info.update(cpp_info, ref.name) + ref = ConanFileReference.loads("My.Fancy-Pkg_2/0.1@user/testing") + folder2 = temp_folder() + folder2 = folder2.replace("\\", "/") + os.makedirs(os.path.join(folder2, "include")) + os.makedirs(os.path.join(folder2, "lib")) + cpp_info = CppInfo(folder2) + conanfile.deps_cpp_info.update(cpp_info, ref.name) + generator = VisualStudioLegacyGenerator(conanfile) + + content = generator.content + xml.etree.ElementTree.fromstring(content) + + self.assertIn('AdditionalIncludeDirectories=""%s/include";"%s/include";"' + % (folder1, folder2), content) + self.assertIn('AdditionalLibraryDirectories=""%s/lib";"%s/lib";"' + % (folder1, folder2), content) diff --git a/conans/test/generators/visual_studio_test.py b/conans/test/generators/visual_studio_test.py index 960a3ee3351..0f43200ec8e 100644 --- a/conans/test/generators/visual_studio_test.py +++ b/conans/test/generators/visual_studio_test.py @@ -1,4 +1,3 @@ -import re import unittest import xml.etree.ElementTree @@ -6,13 +5,13 @@ from conans.model.settings import Settings from conans.model.conan_file import ConanFile -from conans.client.generators.cmake import CMakeGenerator from conans.model.build_info import CppInfo from conans.model.ref import ConanFileReference class VisualStudioGeneratorTest(unittest.TestCase): - def _createInfo(self): + + def valid_xml_test(self): conanfile = ConanFile(None, None, Settings({}), None) ref = ConanFileReference.loads("MyPkg/0.1@user/testing") cpp_info = CppInfo("dummy_root_folder1") @@ -21,19 +20,10 @@ def _createInfo(self): cpp_info = CppInfo("dummy_root_folder2") conanfile.deps_cpp_info.update(cpp_info, ref.name) generator = VisualStudioGenerator(conanfile) - return generator.content - def valid_xml_test(self): - data = self._createInfo() - try: - xml.etree.ElementTree.fromstring(data) - except xml.etree.ElementTree.ParseError as err: - self.fail("Visual studio generated code is not valid! Error %s:\n%s " % (str(err), data)) + content = generator.content + xml.etree.ElementTree.fromstring(content) - - def variables_setup_test(self): - content = self._createInfo() self.assertIn('', content) self.assertIn("dummy_root_folder1", content) self.assertIn("dummy_root_folder2", content) - diff --git a/contributors.txt b/contributors.txt index 8a088adb60c..5f039fc93b1 100644 --- a/contributors.txt +++ b/contributors.txt @@ -20,3 +20,4 @@ Many thanks to all of them! - Ray, Chris (chris@xaltotun.com) - Ries, Uilian (uilianries@gmail.com, @uilianries) - Sechet, Olivier (osechet@gmail.com) +- Sturm, Fabian (f@rtfs.org, @sturmf) From 5707838fe354bababe580e23b61694ec4e0b9c37 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 13:01:43 +0200 Subject: [PATCH 18/39] server fix and using 127.0.0.1 speeds win rest_api_test (#1740) * server fix and using 127.0.0.1 speeds win rest_api_test * fixed broken test --- .ci/travis/run.sh | 2 +- .../rest/controllers/file_upload_download_controller.py | 2 -- conans/server/service/service.py | 2 +- conans/test/remote/auth_bearer_test.py | 7 +++---- conans/test/remote/rest_api_test.py | 7 +++---- conans/test/server/utils/server_launcher.py | 1 + 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 15efea3a88f..3d55b03c915 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -10,4 +10,4 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv activate conan fi -nosetests --with-coverage conans.test +nosetests --with-coverage conans.test --verbosity=2 diff --git a/conans/server/rest/controllers/file_upload_download_controller.py b/conans/server/rest/controllers/file_upload_download_controller.py index 7552f41a994..b49091a0798 100644 --- a/conans/server/rest/controllers/file_upload_download_controller.py +++ b/conans/server/rest/controllers/file_upload_download_controller.py @@ -4,7 +4,6 @@ import os from unicodedata import normalize import six -from conans.errors import NotFoundException class FileUploadDownloadController(Controller): @@ -35,7 +34,6 @@ def put(filepath): abs_path = os.path.abspath(os.path.join(storage_path, os.path.normpath(filepath))) # Body is a stringIO (generator) service.put_file(file_saver, abs_path, token, request.content_length) - return class ConanFileUpload(FileUpload): diff --git a/conans/server/service/service.py b/conans/server/service/service.py index 238df10bf09..592a85430f8 100644 --- a/conans/server/service/service.py +++ b/conans/server/service/service.py @@ -47,7 +47,7 @@ def put_file(self, file_saver, abs_filepath, token, upload_size): file_saver.save(os.path.dirname(abs_filepath)) except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError): - return NotFoundException("File not found") + raise NotFoundException("File not found") def _valid_path(self, filepath, encoded_path): if encoded_path == filepath: diff --git a/conans/test/remote/auth_bearer_test.py b/conans/test/remote/auth_bearer_test.py index 43d481408f5..1f53f7b8398 100644 --- a/conans/test/remote/auth_bearer_test.py +++ b/conans/test/remote/auth_bearer_test.py @@ -51,7 +51,6 @@ class AuthorizeBearerTest(unittest.TestCase): def basic_test(self): auth = AuthorizationHeaderSpy() server = TestServer(plugins=[auth]) - server.app servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": conanfile}) @@ -77,13 +76,13 @@ def no_signature_test(self): auth = AuthorizationHeaderSpy() retur = ReturnHandlerPlugin() server = TestServer(plugins=[auth, retur]) - server.app servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client.save({"conanfile.py": conanfile}) client.run("export lasote/stable") - errors = client.run("upload Hello/0.1@lasote/stable") - self.assertFalse(errors) + # Upload will fail, as conan_server is expecting a signed URL + errors = client.run("upload Hello/0.1@lasote/stable", ignore_error=True) + self.assertTrue(errors) expected_calls = [('get_conan_digest_url', None), ('check_credentials', None), diff --git a/conans/test/remote/rest_api_test.py b/conans/test/remote/rest_api_test.py index 44550528ed1..f2c15393d37 100644 --- a/conans/test/remote/rest_api_test.py +++ b/conans/test/remote/rest_api_test.py @@ -3,8 +3,6 @@ from conans.model.ref import ConanFileReference, PackageReference from conans.test.utils.test_files import hello_source_files from conans.paths import CONANFILE, CONAN_MANIFEST, CONANINFO -import sys -from conans.client.output import ConanOutput, Color from conans.model.info import ConanInfo from conans.test.server.utils.server_launcher import TestServerLauncher import requests @@ -16,6 +14,7 @@ from conans.util.files import md5, save from conans.model.manifest import FileTreeManifest from nose.plugins.attrib import attr +from conans.test.utils.tools import TestBufferConanOutput @attr('slow') @@ -34,8 +33,8 @@ def setUpClass(cls): plugins=[plugin]) cls.server.start() - cls.api = RestApiClient(ConanOutput(sys.stdout, Color), requester=requests) - cls.api.remote_url = "http://localhost:%s" % str(cls.server.port) + cls.api = RestApiClient(TestBufferConanOutput(), requester=requests) + cls.api.remote_url = "http://127.0.0.1:%s" % str(cls.server.port) # Authenticate user token = cls.api.authenticate("private_user", "private_pass") diff --git a/conans/test/server/utils/server_launcher.py b/conans/test/server/utils/server_launcher.py index 0f29456b429..cc180212390 100644 --- a/conans/test/server/utils/server_launcher.py +++ b/conans/test/server/utils/server_launcher.py @@ -119,6 +119,7 @@ def clean(self): if os.path.exists(self.storage_folder): shutil.rmtree(self.storage_folder) + if __name__ == "__main__": server = TestServerLauncher() server.start(daemon=False) From 2ee2304629feb2b5dccec44275e4f7a47f745543 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 13:45:37 +0200 Subject: [PATCH 19/39] Conan config install command (#1705) * initial ideas for config install * adding git config install * added conan.conf * fixed conan.conf set item for py3 * custom files * improved conan config install * use RemoteRegistry to parse remotes --- conans/client/command.py | 8 +- conans/client/conan_api.py | 5 + conans/client/conf/config_installer.py | 108 ++++++++++++ conans/client/remote_registry.py | 9 + conans/test/command/config_install_test.py | 187 +++++++++++++++++++++ conans/test/functional/cmake_test.py | 4 +- conans/test/util/output_test.py | 2 +- conans/test/util/tools_test.py | 2 +- conans/tools.py | 5 +- 9 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 conans/client/conf/config_installer.py create mode 100644 conans/test/command/config_install_test.py diff --git a/conans/client/command.py b/conans/client/command.py index da3378cbbc0..f22dd222f9c 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -268,7 +268,7 @@ def install(self, *args): filename=args.file, cwd=args.cwd) def config(self, *args): - """Manages conan.conf information + """Manages conan configuration information """ parser = argparse.ArgumentParser(description=self.config.__doc__, prog="conan config") @@ -276,11 +276,13 @@ def config(self, *args): rm_subparser = subparsers.add_parser('rm', help='rm an existing config element') set_subparser = subparsers.add_parser('set', help='set/add value') get_subparser = subparsers.add_parser('get', help='get the value of existing element') + install_subparser = subparsers.add_parser('install', + help='install a full configuration from a zip file, local or remote') rm_subparser.add_argument("item", help="item to remove") get_subparser.add_argument("item", nargs="?", help="item to print") set_subparser.add_argument("item", help="key=value to set") - + install_subparser.add_argument("item", nargs="?", help="configuration file to use") args = parser.parse_args(*args) if args.subcommand == "set": @@ -293,6 +295,8 @@ def config(self, *args): return self._conan.config_get(args.item) elif args.subcommand == "rm": return self._conan.config_rm(args.item) + elif args.subcommand == "install": + return self._conan.config_install(args.item) def info(self, *args): """Prints information about a package recipe's dependency graph. diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index dbc384e5c07..1a6ef8b2360 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -414,6 +414,11 @@ def config_rm(self, item): config_parser = ConanClientConfigParser(self._client_cache.conan_conf_path) config_parser.rm_item(item) + @api_method + def config_install(self, item): + from conans.client.conf.config_installer import configuration_install + return configuration_install(item, self._client_cache, self._user_io.out, self._runner) + @api_method def info_build_order(self, reference, settings=None, options=None, env=None, scope=None, profile_name=None, filename=None, remote=None, build_order=None, check_updates=None, cwd=None): diff --git a/conans/client/conf/config_installer.py b/conans/client/conf/config_installer.py new file mode 100644 index 00000000000..b31e8ef9f4f --- /dev/null +++ b/conans/client/conf/config_installer.py @@ -0,0 +1,108 @@ +import os +from conans.tools import unzip +import shutil +from conans.util.files import rmdir, mkdir +from conans.client.remote_registry import RemoteRegistry +from conans import tools +from conans.errors import ConanException + + +def _handle_remotes(registry_path, remote_file, output): + registry = RemoteRegistry(registry_path, output) + new_registry = RemoteRegistry(remote_file, output) + registry.define_remotes(new_registry.remotes) + + +def _handle_profiles(source_folder, target_folder, output): + mkdir(target_folder) + for root, _, files in os.walk(source_folder): + relative_path = os.path.relpath(root, source_folder) + if relative_path == ".": + relative_path = "" + for f in files: + profile = os.path.join(relative_path, f) + output.info(" Installing profile %s" % profile) + shutil.copy(os.path.join(root, f), os.path.join(target_folder, profile)) + + +def _process_git_repo(repo_url, client_cache, output, runner, tmp_folder): + output.info("Trying to clone repo %s" % repo_url) + + with tools.chdir(tmp_folder): + runner('git clone "%s" config' % repo_url, output=output) + tmp_folder = os.path.join(tmp_folder, "config") + _process_folder(tmp_folder, client_cache, output) + + +def _process_zip_file(zippath, client_cache, output, tmp_folder): + unzip(zippath, tmp_folder) + os.unlink(zippath) + _process_folder(tmp_folder, client_cache, output) + + +def _handle_conan_conf(current_conan_conf, new_conan_conf_path): + current_conan_conf.read(new_conan_conf_path) + with open(current_conan_conf.filename, "w") as f: + current_conan_conf.write(f) + + +def _process_folder(folder, client_cache, output): + for root, dirs, files in os.walk(folder): + for f in files: + if f == "settings.yml": + output.info("Installing settings.yml") + settings_path = client_cache.settings_path + shutil.copy(os.path.join(root, f), settings_path) + elif f == "conan.conf": + output.info("Processing conan.conf") + conan_conf = client_cache.conan_config + _handle_conan_conf(conan_conf, os.path.join(root, f)) + elif f == "remotes.txt": + output.info("Defining remotes") + registry_path = client_cache.registry + _handle_remotes(registry_path, os.path.join(root, f), output) + else: + output.info("Copying file %s to %s" % (f, client_cache.conan_folder)) + shutil.copy(os.path.join(root, f), client_cache.conan_folder) + for d in dirs: + if d == "profiles": + output.info("Installing profiles") + profiles_path = client_cache.profiles_path + _handle_profiles(os.path.join(root, d), profiles_path, output) + break + dirs[:] = [d for d in dirs if d not in ("profiles", ".git")] + + +def _process_download(item, client_cache, output, tmp_folder): + output.info("Trying to download %s" % item) + zippath = os.path.join(tmp_folder, "config.zip") + tools.download(item, zippath, out=output) + _process_zip_file(zippath, client_cache, output, tmp_folder) + + +def configuration_install(item, client_cache, output, runner): + tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install") + # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/ + tmp_folder = os.path.realpath(tmp_folder) + mkdir(tmp_folder) + try: + if item is None: + try: + item = client_cache.conan_config.get_item("general.config_install") + except ConanException: + raise ConanException("Called config install without arguments and " + "'general.config_install' not defined in conan.conf") + + if item.endswith(".git"): + _process_git_repo(item, client_cache, output, runner, tmp_folder) + elif os.path.exists(item): + # is a local file + _process_zip_file(item, client_cache, output, tmp_folder) + elif item.startswith("http"): + _process_download(item, client_cache, output, tmp_folder) + else: + raise ConanException("I don't know how to process %s" % item) + finally: + if item: + client_cache.conan_config.set_item("general.config_install", item) + rmdir(tmp_folder) diff --git a/conans/client/remote_registry.py b/conans/client/remote_registry.py index 534d5576761..b6ebed6943d 100644 --- a/conans/client/remote_registry.py +++ b/conans/client/remote_registry.py @@ -173,6 +173,15 @@ def exists_function(remotes): raise ConanException("Remote '%s' not found in remotes" % remote_name) self._add_update(remote_name, remote, verify_ssl, exists_function, insert) + def define_remotes(self, remotes): + with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): + _, refs = self._load() + new_remotes = OrderedDict() + for remote in remotes: + new_remotes[remote.name] = (remote.url, remote.verify_ssl) + refs = {k: v for k, v in refs.items() if v in new_remotes} + self._save(new_remotes, refs) + def _add_update(self, remote_name, remote, verify_ssl, exists_function, insert=None): with fasteners.InterProcessLock(self._filename + ".lock", logger=logger): diff --git a/conans/test/command/config_install_test.py b/conans/test/command/config_install_test.py new file mode 100644 index 00000000000..983e90999c0 --- /dev/null +++ b/conans/test/command/config_install_test.py @@ -0,0 +1,187 @@ +import unittest + +from conans.test.utils.tools import TestClient, TestBufferConanOutput +import os +import zipfile +from conans.test.utils.test_files import temp_folder +from conans.util.files import load, save_files, save +from conans.client.remote_registry import RemoteRegistry, Remote +from mock import patch +from conans.client.rest.uploader_downloader import Downloader +from conans import tools +from conans.client.conf import ConanClientConfigParser +import shutil + + +win_profile = """[settings] + os: Windows +""" + +linux_profile = """[settings] + os: Linux +""" + +remotes = """myrepo1 https://myrepourl.net False +my-repo-2 https://myrepo2.com True +""" + +registry = """myrepo1 https://myrepourl.net False + +Pkg/1.0@user/channel myrepo1 +""" + +settings_yml = """os: + Windows: + Linux: +arch: [x86, x86_64] +""" + +conan_conf = """ +[log] +run_to_output = False # environment CONAN_LOG_RUN_TO_OUTPUT +level = 10 # environment CONAN_LOGGING_LEVEL + +[general] +compression_level = 6 # environment CONAN_COMPRESSION_LEVEL +cpu_count = 1 # environment CONAN_CPU_COUNT + +[proxies] +# Empty section will try to use system proxies. +# If don't want proxy at all, remove section [proxies] +# As documented in http://docs.python-requests.org/en/latest/user/advanced/#proxies +http = http://user:pass@10.10.1.10:3128/ +no_proxy = mylocalhost +https = None +# http = http://10.10.1.10:3128 +# https = http://10.10.1.10:1080 +""" + + +def zipdir(path, zipfilename): + with zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_DEFLATED) as z: + for root, _, files in os.walk(path): + for f in files: + z.write(os.path.join(root, f)) + + +class ConfigInstallTest(unittest.TestCase): + + def setUp(self): + self.client = TestClient() + registry_path = self.client.client_cache.registry + + save(registry_path, """my-repo-2 https://myrepo2.com True +conan-center https://conan-center.com + +MyPkg/0.1@user/channel my-repo-2 +Other/1.2@user/channel conan-center +""") + save(os.path.join(self.client.client_cache.profiles_path, "default"), "#default profile empty") + save(os.path.join(self.client.client_cache.profiles_path, "linux"), "#empty linux profile") + + def _create_profile_folder(self, folder=None): + folder = folder or temp_folder(path_with_spaces=False) + save_files(folder, {"settings.yml": settings_yml, + "remotes.txt": remotes, + "profiles/linux": linux_profile, + "profiles/windows": win_profile, + "config/conan.conf": conan_conf, + "pylintrc": "#Custom pylint"}) + return folder + + def _create_zip(self, zippath=None): + folder = self._create_profile_folder() + zippath = zippath or os.path.join(folder, "myconfig.zip") + zipdir(folder, zippath) + return zippath + + def _check(self, install_path): + settings_path = self.client.client_cache.settings_path + self.assertEqual(load(settings_path).splitlines(), settings_yml.splitlines()) + registry_path = self.client.client_cache.registry + registry = RemoteRegistry(registry_path, TestBufferConanOutput()) + self.assertEqual(registry.remotes, + [Remote("myrepo1", "https://myrepourl.net", False), + Remote("my-repo-2", "https://myrepo2.com", True), + ]) + self.assertEqual(registry.refs, {"MyPkg/0.1@user/channel": "my-repo-2"}) + self.assertEqual(sorted(os.listdir(self.client.client_cache.profiles_path)), + sorted(["default", "linux", "windows"])) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "linux")).splitlines(), + linux_profile.splitlines()) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "windows")).splitlines(), + win_profile.splitlines()) + conan_conf = ConanClientConfigParser(self.client.client_cache.conan_conf_path) + self.assertEqual(conan_conf.get_item("log.run_to_output"), "False") + self.assertEqual(conan_conf.get_item("log.run_to_file"), "False") + self.assertEqual(conan_conf.get_item("log.level"), "10") + self.assertEqual(conan_conf.get_item("general.compression_level"), "6") + self.assertEqual(conan_conf.get_item("general.sysrequires_sudo"), "True") + self.assertEqual(conan_conf.get_item("general.cpu_count"), "1") + self.assertEqual(conan_conf.get_item("general.config_install"), install_path) + self.assertEqual(conan_conf.get_item("proxies.no_proxy"), "mylocalhost") + self.assertEqual(conan_conf.get_item("proxies.https"), "None") + self.assertEqual(conan_conf.get_item("proxies.http"), "http://user:pass@10.10.1.10:3128/") + self.assertEqual("#Custom pylint", + load(os.path.join(self.client.client_cache.conan_folder, "pylintrc"))) + + def install_file_test(self): + """ should install from a file in current dir + """ + zippath = self._create_zip() + self.client.run('config install "%s"' % zippath) + self._check(zippath) + + def test_without_profile_folder(self): + shutil.rmtree(self.client.client_cache.profiles_path) + zippath = self._create_zip() + self.client.run('config install "%s"' % zippath) + self.assertEqual(sorted(os.listdir(self.client.client_cache.profiles_path)), + sorted(["linux", "windows"])) + self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "linux")).splitlines(), + linux_profile.splitlines()) + + def install_url_test(self): + """ should install from a URL + """ + + def my_download(obj, url, filename, **kwargs): # @UnusedVariable + self._create_zip(filename) + + with patch.object(Downloader, 'download', new=my_download): + self.client.run("config install http://myfakeurl.com/myconf.zip") + self._check("http://myfakeurl.com/myconf.zip") + + # repeat the process to check + self.client.run("config install http://myfakeurl.com/myconf.zip") + self._check("http://myfakeurl.com/myconf.zip") + + def install_repo_test(self): + """ should install from a git repo + """ + + folder = self._create_profile_folder() + with tools.chdir(folder): + self.client.runner('git init .') + self.client.runner('git add .') + self.client.runner('git config user.name myname') + self.client.runner('git config user.email myname@mycompany.com') + self.client.runner('git commit -m "mymsg"') + + self.client.run('config install "%s/.git"' % folder) + self._check("%s/.git" % folder) + + def reinstall_test(self): + """ should use configured URL in conan.conf + """ + zippath = self._create_zip() + self.client.run('config set general.config_install="%s"' % zippath) + self.client.run("config install") + self._check(zippath) + + def reinstall_error_test(self): + """ should use configured URL in conan.conf + """ + error = self.client.run("config install", ignore_error=True) + self.assertTrue(error) + self.assertIn("Called config install without arguments", self.client.out) diff --git a/conans/test/functional/cmake_test.py b/conans/test/functional/cmake_test.py index b7fdf9726e7..564d9c50982 100644 --- a/conans/test/functional/cmake_test.py +++ b/conans/test/functional/cmake_test.py @@ -1,7 +1,6 @@ import os import shutil import sys -import tempfile import unittest import platform @@ -15,11 +14,12 @@ from conans.test.utils.tools import TestBufferConanOutput from conans.tools import cpu_count from conans.util.files import save +from conans.test.utils.test_files import temp_folder class CMakeTest(unittest.TestCase): def setUp(self): - self.tempdir = tempfile.mkdtemp() + self.tempdir = temp_folder(path_with_spaces=False) def tearDown(self): shutil.rmtree(self.tempdir) diff --git a/conans/test/util/output_test.py b/conans/test/util/output_test.py index ce34ccfc6f0..eba9e2afb97 100644 --- a/conans/test/util/output_test.py +++ b/conans/test/util/output_test.py @@ -71,6 +71,6 @@ def unzip_output_test(self): sys.stdout = old_out output = new_out.getvalue() - self.assertRegexpMatches(output, "Unzipping [\d]+B, this can take a while") + self.assertRegexpMatches(output, "Unzipping [\d]+B") content = load(os.path.join(output_dir, "example.txt")) self.assertEqual(content, "Hello world!") diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 3fcf13b3572..4af2636ca7d 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -95,7 +95,7 @@ def build(self): # Not test the real commmand get_command if it's setting the module global vars tools._global_requester = None tools._global_output = None - tmp = tempfile.mkdtemp() + tmp = temp_folder() conf = default_client_conf.replace("\n[proxies]", "\n[proxies]\nhttp = http://myproxy.com") os.mkdir(os.path.join(tmp, ".conan")) save(os.path.join(tmp, ".conan", CONAN_CONF), conf) diff --git a/conans/tools.py b/conans/tools.py index c84f2aba9f8..3c014f61272 100644 --- a/conans/tools.py +++ b/conans/tools.py @@ -292,7 +292,10 @@ def print_progress(extracted_size, uncompress_size): with zipfile.ZipFile(filename, "r") as z: uncompress_size = sum((file_.file_size for file_ in z.infolist())) - _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) + if uncompress_size > 100000: + _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) + else: + _global_output.info("Unzipping %s" % human_size(uncompress_size)) extracted_size = 0 if platform.system() == "Windows": for file_ in z.infolist(): From 5bc571a46b9c2f28b7658f3ff0525ac3e8fb8f6f Mon Sep 17 00:00:00 2001 From: zomeck Date: Tue, 12 Sep 2017 14:03:00 +0200 Subject: [PATCH 20/39] Change default arguments for manifests related arguments. (#1731) Throw Exception when manifest_folder is relative and cwd is not defined. --- conans/client/conan_api.py | 14 ++-- conans/test/conan_api/__init__.py | 0 .../conan_api/manifests_arguments_test.py | 71 +++++++++++++++++++ 3 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 conans/test/conan_api/__init__.py create mode 100644 conans/test/conan_api/manifests_arguments_test.py diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 1a6ef8b2360..a420220e598 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -168,8 +168,8 @@ def new(self, name, header=False, pure_c=False, test=False, exports_sources=Fals @api_method def test_package(self, profile_name=None, settings=None, options=None, env=None, scope=None, test_folder=None, not_export=False, build=None, keep_source=False, - verify=default_manifest_folder, manifests=default_manifest_folder, - manifests_interactive=default_manifest_folder, + verify=None, manifests=None, + manifests_interactive=None, remote=None, update=False, cwd=None, user=None, channel=None, name=None, version=None): settings = settings or [] @@ -260,8 +260,8 @@ def test_package(self, profile_name=None, settings=None, options=None, env=None, @api_method def create(self, profile_name=None, settings=None, options=None, env=None, scope=None, test_folder=None, not_export=False, build=None, - keep_source=False, verify=default_manifest_folder, - manifests=default_manifest_folder, manifests_interactive=default_manifest_folder, + keep_source=False, verify=None, + manifests=None, manifests_interactive=None, remote=None, update=False, cwd=None, user=None, channel=None, name=None, version=None): @@ -359,8 +359,8 @@ def package_files(self, reference, source_folder=None, build_folder=None, packag @api_method def install(self, reference="", package=None, settings=None, options=None, env=None, scope=None, all=False, - remote=None, werror=False, verify=default_manifest_folder, manifests=default_manifest_folder, - manifests_interactive=default_manifest_folder, build=None, profile_name=None, + remote=None, werror=False, verify=None, manifests=None, + manifests_interactive=None, build=None, profile_name=None, update=False, generator=None, no_imports=False, filename=None, cwd=None): self._user_io.out.werror_active = werror @@ -763,6 +763,8 @@ def _parse_manifests_arguments(verify, manifests, manifests_interactive, cwd): manifest_folder = verify or manifests or manifests_interactive if manifest_folder: if not os.path.isabs(manifest_folder): + if not cwd: + raise ConanException("'cwd' should be defined if the manifest folder is relative.") manifest_folder = os.path.join(cwd, manifest_folder) manifest_verify = verify is not None manifest_interactive = manifests_interactive is not None diff --git a/conans/test/conan_api/__init__.py b/conans/test/conan_api/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/conans/test/conan_api/manifests_arguments_test.py b/conans/test/conan_api/manifests_arguments_test.py new file mode 100644 index 00000000000..0c315c7dc2e --- /dev/null +++ b/conans/test/conan_api/manifests_arguments_test.py @@ -0,0 +1,71 @@ +from conans.client.conan_api import _parse_manifests_arguments, ConanException, default_manifest_folder, prepare_cwd +import unittest +from nose_parameterized.parameterized import parameterized + + +class ArgumentsTest(unittest.TestCase): + @parameterized.expand([ + (dict(verify=default_manifest_folder, + manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder),), + (dict(verify=None, + manifests=default_manifest_folder, + manifests_interactive=default_manifest_folder),), + (dict(verify=default_manifest_folder, + manifests=None, + manifests_interactive=default_manifest_folder),), + (dict(verify=default_manifest_folder, + manifests=default_manifest_folder, + manifests_interactive=None),), + (dict(verify=default_manifest_folder, + manifests=None, + manifests_interactive=None),), + ]) + def test_manifest_arguments_conflicting(self, arguments): + with self.assertRaises(ConanException): + _parse_manifests_arguments(cwd=None, **arguments) + + def test_manifests_arguments_verify(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=default_manifest_folder, + manifests=None, + manifests_interactive=None, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertFalse(manifest_interactive) + self.assertTrue(manifest_verify) + + def test_manifests_arguments_manifests_interactive(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, + manifests=None, + manifests_interactive=default_manifest_folder, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertTrue(manifest_interactive) + self.assertFalse(manifest_verify) + + def test_manifests_arguments_manifests(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, + manifests=default_manifest_folder, + manifests_interactive=None, + cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIn(cwd, manifest_folder) + self.assertFalse(manifest_interactive) + self.assertFalse(manifest_verify) + + def test_manifests_arguments_no_manifests(self): + cwd = prepare_cwd(None) + manifests = _parse_manifests_arguments(verify=None, manifests=None, manifests_interactive=None, cwd=cwd) + manifest_folder, manifest_interactive, manifest_verify = manifests + + self.assertIsNone(manifest_folder) + self.assertFalse(manifest_interactive) + self.assertFalse(manifest_verify) From 949cdec3ce01382306e12b62086740fe04490ddb Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Sep 2017 14:48:34 +0200 Subject: [PATCH 21/39] making short paths configurable in Win with env-var CONAN_USER_HOME_SHORT (#1725) * making short paths configurable in Win with env-var CONAN_USER_HOME_SHORT * fixed path_shortener noop --- conans/client/client_cache.py | 6 +- conans/client/conf/__init__.py | 1 + conans/client/installer.py | 24 ++---- conans/paths.py | 99 ++++------------------- conans/test/functional/path_limit_test.py | 33 ++++++++ conans/util/windows.py | 83 +++++++++++++++++++ 6 files changed, 139 insertions(+), 107 deletions(-) create mode 100644 conans/util/windows.py diff --git a/conans/client/client_cache.py b/conans/client/client_cache.py index 7b78f0eeac5..73219dd7032 100644 --- a/conans/client/client_cache.py +++ b/conans/client/client_cache.py @@ -65,15 +65,11 @@ def registry(self): @property def conan_config(self): - def generate_default_config_file(): - save(self.conan_conf_path, normalize(default_client_conf)) - if not self._conan_config: if not os.path.exists(self.conan_conf_path): - generate_default_config_file() + save(self.conan_conf_path, normalize(default_client_conf)) self._conan_config = ConanClientConfigParser(self.conan_conf_path) - return self._conan_config @property diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 1f87bc1118e..6bbf43a4f74 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -124,6 +124,7 @@ def env_vars(self): "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_RECIPE_LINTER": self._env_c("general.recipe_linter", "CONAN_RECIPE_LINTER", "True"), "CONAN_CPU_COUNT": self._env_c("general.cpu_count", "CONAN_CPU_COUNT", None), + "CONAN_USER_HOME_SHORT": self._env_c("general.user_home_short", "CONAN_USER_HOME_SHORT", None), "CONAN_VERBOSE_TRACEBACK": self._env_c("general.verbose_traceback", "CONAN_VERBOSE_TRACEBACK", None), # http://www.vtk.org/Wiki/CMake_Cross_Compiling "CONAN_CMAKE_GENERATOR": self._env_c("general.cmake_generator", "CONAN_CMAKE_GENERATOR", None), diff --git a/conans/client/installer.py b/conans/client/installer.py index cddc1b367ae..671c2d5c00b 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -123,27 +123,17 @@ def _build_package(self): self._conan_file, self._out) self._out.info('Copying sources to build folder') - def check_max_path_len(src, files): - if platform.system() != "Windows": - return [] - filtered_files = [] - for the_file in files: - source_path = os.path.join(src, the_file) - # Without storage path, just relative - rel_path = os.path.relpath(source_path, src_folder) - dest_path = os.path.normpath(os.path.join(self.build_folder, rel_path)) - # it is NOT that "/" is counted as "\\" so it counts double - # seems a bug in python, overflows paths near the limit of 260, - if len(dest_path) >= 249: - filtered_files.append(the_file) - self._out.warn("Filename too long, file excluded: %s" % dest_path) - return filtered_files - if getattr(self._conan_file, 'no_copy_source', False): mkdir(self.build_folder) self._conan_file.source_folder = src_folder else: - shutil.copytree(src_folder, self.build_folder, symlinks=True, ignore=check_max_path_len) + if platform.system() == "Windows" and os.getenv("CONAN_USER_HOME_SHORT") != "None": + from conans.util.windows import ignore_long_path_files + ignore = ignore_long_path_files(src_folder, self.build_folder, self._out) + else: + ignore = None + + shutil.copytree(src_folder, self.build_folder, symlinks=True, ignore=ignore) logger.debug("Copied to %s" % self.build_folder) logger.debug("Files copied %s" % os.listdir(self.build_folder)) self._conan_file.source_folder = self.build_folder diff --git a/conans/paths.py b/conans/paths.py index 9f89e342736..a3dd696d885 100644 --- a/conans/paths.py +++ b/conans/paths.py @@ -1,10 +1,17 @@ import os from conans.model.ref import ConanFileReference, PackageReference -from conans.util.files import load, save, rmdir from os.path import join, normpath import platform -import tempfile from conans.errors import ConanException +from conans.util.files import rmdir + +if platform.system() == "Windows": + from conans.util.windows import path_shortener, rm_conandir, conan_expand_user +else: + def path_shortener(x, _): + return x + conan_expand_user = os.path.expanduser + rm_conandir = rmdir EXPORT_FOLDER = "export" @@ -36,39 +43,13 @@ EXPORT_TGZ_NAME = "conan_export.tgz" EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz" EXPORT_SOURCES_DIR = ".c_src" -CONAN_LINK = ".conan_link" + RUN_LOG_NAME = "conan_run.log" DEFAULT_PROFILE_NAME = "default" -def conan_expand_user(path): - """ wrapper to the original expanduser function, to workaround python returning - verbatim %USERPROFILE% when some other app (git for windows) sets HOME envvar - """ - if platform.system() == "Windows": - # In win these variables should exist and point to user directory, which - # must exist. Using context to avoid permanent modification of os.environ - old_env = dict(os.environ) - try: - home = os.environ.get("HOME") - # Problematic cases of wrong HOME variable - # - HOME = %USERPROFILE% verbatim, as messed by some other tools - # - MSYS console, that defines a different user home in /c/mingw/msys/users/xxx - # In these cases, it is safe to remove it and rely on USERPROFILE directly - if home and (not os.path.exists(home) or - (os.getenv("MSYSTEM") and os.getenv("USERPROFILE"))): - del os.environ["HOME"] - result = os.path.expanduser(path) - finally: - os.environ.clear() - os.environ.update(old_env) - return result - - return os.path.expanduser(path) - - def get_conan_user_home(): tmp = conan_expand_user(os.getenv("CONAN_USER_HOME", "~")) if not os.path.isabs(tmp): @@ -78,19 +59,6 @@ def get_conan_user_home(): return os.path.abspath(tmp) -if platform.system() == "Windows": - def _rm_conandir(path): - """removal of a directory that might contain a link to a short path""" - link = os.path.join(path, CONAN_LINK) - if os.path.exists(link): - short_path = load(link) - rmdir(os.path.dirname(short_path)) - rmdir(path) - rm_conandir = _rm_conandir -else: - rm_conandir = rmdir - - def is_case_insensitive_os(): system = platform.system() return system != "Linux" and system != "FreeBSD" and system != "SunOS" @@ -119,41 +87,6 @@ def _check_ref_case(conan_reference, conan_folder, store_folder): # @UnusedVari pass -def _shortener(path, short_paths): - """ short_paths is 4-state: - False: Never shorten the path - True: Always shorten the path, create link if not existing - None: Use shorten path only if already exists, not create - Other: Integrity check. Consumer knows it should be short, but it isn't - """ - if short_paths is False: - return path - link = os.path.join(path, CONAN_LINK) - if os.path.exists(link): - return load(link) - elif short_paths is None: - return path - elif short_paths is not True: - raise ConanException("This path should be short, but it isn't: %s\n" - "Try to remove these packages and re-build them" % path) - - short_home = os.getenv("CONAN_USER_HOME_SHORT") - if not short_home: - drive = os.path.splitdrive(path)[0] - short_home = drive + "/.conan" - try: - os.makedirs(short_home) - except: - pass - redirect = tempfile.mkdtemp(dir=short_home, prefix="") - # This "1" is the way to have a non-existing directory, so commands like - # shutil.copytree() to it, works. It can be removed without compromising the - # temp folder generator and conan-links consistency - redirect = os.path.join(redirect, "1") - save(link, redirect) - return redirect - - class SimplePaths(object): """ Generate Conan paths. Handles the conan domain path logic. NO DISK ACCESS, just @@ -161,10 +94,6 @@ class SimplePaths(object): """ def __init__(self, store_folder): self._store_folder = store_folder - if platform.system() == "Windows": - self._shortener = _shortener - else: - self._shortener = lambda x, _: x @property def store(self): @@ -187,12 +116,12 @@ def export(self, conan_reference): def export_sources(self, conan_reference, short_paths=False): assert isinstance(conan_reference, ConanFileReference) p = normpath(join(self.conan(conan_reference), EXPORT_SRC_FOLDER)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def source(self, conan_reference, short_paths=False): assert isinstance(conan_reference, ConanFileReference) p = normpath(join(self.conan(conan_reference), SRC_FOLDER)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def conanfile(self, conan_reference): export = self.export(conan_reference) @@ -216,7 +145,7 @@ def build(self, package_reference, short_paths=False): assert isinstance(package_reference, PackageReference) p = normpath(join(self.conan(package_reference.conan), BUILD_FOLDER, package_reference.package_id)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) def system_reqs(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) @@ -235,4 +164,4 @@ def package(self, package_reference, short_paths=False): assert isinstance(package_reference, PackageReference) p = normpath(join(self.conan(package_reference.conan), PACKAGES_FOLDER, package_reference.package_id)) - return self._shortener(p, short_paths) + return path_shortener(p, short_paths) diff --git a/conans/test/functional/path_limit_test.py b/conans/test/functional/path_limit_test.py index 3fa19308690..a7eab70fba9 100644 --- a/conans/test/functional/path_limit_test.py +++ b/conans/test/functional/path_limit_test.py @@ -196,6 +196,39 @@ def basic_test(self): self.assertFalse(os.path.exists(link_build)) self.assertFalse(os.path.exists(link_package)) + def basic_disabled_test(self): + client = TestClient() + base = ''' +from conans import ConanFile + +class ConanLib(ConanFile): + short_paths = True +''' + files = {"conanfile.py": base} + client.save(files) + client.client_cache.conan_config.set_item("general.user_home_short", "None") + + client.run("create lib/0.1@user/channel") + package_ref = PackageReference.loads("lib/0.1@user/channel:" + "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") + client.run("search") + self.assertIn("lib/0.1@user/channel", client.user_io.out) + client.run("search lib/0.1@user/channel") + self.assertIn("Package_ID: 5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", client.user_io.out) + + conan_ref = ConanFileReference.loads("lib/0.1@user/channel") + source_folder = client.client_cache.source(conan_ref) + link_source = os.path.join(source_folder, ".conan_link") + self.assertFalse(os.path.exists(link_source)) + + build_folder = client.client_cache.build(package_ref) + link_build = os.path.join(build_folder, ".conan_link") + self.assertFalse(os.path.exists(link_build)) + + package_folder = client.client_cache.package(package_ref) + link_package = os.path.join(package_folder, ".conan_link") + self.assertFalse(os.path.exists(link_package)) + def failure_test(self): base = ''' diff --git a/conans/util/windows.py b/conans/util/windows.py new file mode 100644 index 00000000000..72112433ccb --- /dev/null +++ b/conans/util/windows.py @@ -0,0 +1,83 @@ +import os +from conans.util.files import load, mkdir, save, rmdir +import tempfile + + +CONAN_LINK = ".conan_link" + + +def conan_expand_user(path): + """ wrapper to the original expanduser function, to workaround python returning + verbatim %USERPROFILE% when some other app (git for windows) sets HOME envvar + """ + # In win these variables should exist and point to user directory, which + # must exist. Using context to avoid permanent modification of os.environ + old_env = dict(os.environ) + try: + home = os.environ.get("HOME") + # Problematic cases of wrong HOME variable + # - HOME = %USERPROFILE% verbatim, as messed by some other tools + # - MSYS console, that defines a different user home in /c/mingw/msys/users/xxx + # In these cases, it is safe to remove it and rely on USERPROFILE directly + if home and (not os.path.exists(home) or + (os.getenv("MSYSTEM") and os.getenv("USERPROFILE"))): + del os.environ["HOME"] + result = os.path.expanduser(path) + finally: + os.environ.clear() + os.environ.update(old_env) + return result + + +def path_shortener(path, short_paths): + """ short_paths is 4-state: + False: Never shorten the path + True: Always shorten the path, create link if not existing + None: Use shorten path only if already exists, not create + """ + if short_paths is False or os.getenv("CONAN_USER_HOME_SHORT") == "None": + return path + link = os.path.join(path, CONAN_LINK) + if os.path.exists(link): + return load(link) + elif short_paths is None: + return path + + short_home = os.getenv("CONAN_USER_HOME_SHORT") + if not short_home: + drive = os.path.splitdrive(path)[0] + short_home = drive + "/.conan" + mkdir(short_home) + redirect = tempfile.mkdtemp(dir=short_home, prefix="") + # This "1" is the way to have a non-existing directory, so commands like + # shutil.copytree() to it, works. It can be removed without compromising the + # temp folder generator and conan-links consistency + redirect = os.path.join(redirect, "1") + save(link, redirect) + return redirect + + +def ignore_long_path_files(src_folder, build_folder, output): + def _filter(src, files): + filtered_files = [] + for the_file in files: + source_path = os.path.join(src, the_file) + # Without storage path, just relative + rel_path = os.path.relpath(source_path, src_folder) + dest_path = os.path.normpath(os.path.join(build_folder, rel_path)) + # it is NOT that "/" is counted as "\\" so it counts double + # seems a bug in python, overflows paths near the limit of 260, + if len(dest_path) >= 249: + filtered_files.append(the_file) + output.warn("Filename too long, file excluded: %s" % dest_path) + return filtered_files + return _filter + + +def rm_conandir(path): + """removal of a directory that might contain a link to a short path""" + link = os.path.join(path, CONAN_LINK) + if os.path.exists(link): + short_path = load(link) + rmdir(os.path.dirname(short_path)) + rmdir(path) From b0ac3553dbd4d821c9d2ad6c49193456f6c07d2d Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 12 Sep 2017 23:25:42 +0200 Subject: [PATCH 22/39] In local cache flag (#1747) --- conans/client/loader.py | 1 + conans/model/conan_file.py | 3 + conans/test/functional/in_local_cache_test.py | 69 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 conans/test/functional/in_local_cache_test.py diff --git a/conans/client/loader.py b/conans/client/loader.py index c31946cd766..bb20887aeb2 100644 --- a/conans/client/loader.py +++ b/conans/client/loader.py @@ -66,6 +66,7 @@ def load_conan(self, conanfile_path, output, consumer=False, reference=None): result.scope = self._scopes.package_scope() else: result.scope = self._scopes.package_scope(result.name) + result.in_local_cache = True return result except Exception as e: # re-raise with file name diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index a24ce5e93f2..c4665a182ce 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -130,6 +130,9 @@ def __init__(self, output, runner, settings, conanfile_directory, user=None, cha self._user = user self._channel = channel + # Are we in local cache? Suggest a better name + self.in_local_cache = False + @property def env(self): simple, multiple = self._env_values.env_dicts(self.name) diff --git a/conans/test/functional/in_local_cache_test.py b/conans/test/functional/in_local_cache_test.py new file mode 100644 index 00000000000..61971d51790 --- /dev/null +++ b/conans/test/functional/in_local_cache_test.py @@ -0,0 +1,69 @@ +import os +import unittest +from conans.test.utils.tools import TestClient +from conans.paths import CONANFILE + + +conanfile = """ +from conans import ConanFile, tools + +class AConan(ConanFile): + name = "Hello0" + version = "0.1" + + def build(self): + self.output.warn("build() IN LOCAL CACHE=> %s" % str(self.in_local_cache)) + + def package(self): + self.output.warn("package() IN LOCAL CACHE=> %s" % str(self.in_local_cache)) + +""" + + +class InLocalCacheTest(unittest.TestCase): + + def test_in_local_cache_flag(self): + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("export lasote/stable") + client.run("install Hello0/0.1@lasote/stable --build missing") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("install .") + client.run("build") + self.assertIn("build() IN LOCAL CACHE=> False", client.user_io.out) + + pack_folder = os.path.join(client.current_folder, "package") + os.mkdir(pack_folder) + client.current_folder = pack_folder + client.run("package .. --build_folder ..") + self.assertIn("package() IN LOCAL CACHE=> False", client.user_io.out) + + # Confirm that we have the flag depending on the recipe too + client = TestClient() + client.save({CONANFILE: conanfile}) + client.run("export lasote/stable") + conanfile_reuse = """ +from conans import ConanFile, tools + +class OtherConan(ConanFile): + name = "Hello1" + version = "0.1" + requires = "Hello0/0.1@lasote/stable" + + def build(self): + pass +""" + client.save({CONANFILE: conanfile_reuse}, clean_first=True) + client.run("install . --build") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + client.run("export lasote/stable") + client.run("install Hello1/0.1@lasote/stable --build") + self.assertIn("build() IN LOCAL CACHE=> True", client.user_io.out) + self.assertIn("package() IN LOCAL CACHE=> True", client.user_io.out) + + From d621731187d1c18b4bea9e33c9e6f79f7699cfe6 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 13 Sep 2017 09:22:47 +0200 Subject: [PATCH 23/39] new version-range test (#1749) --- conans/client/require_resolver.py | 2 -- conans/test/model/version_ranges_test.py | 11 +++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/conans/client/require_resolver.py b/conans/client/require_resolver.py index fa0896b722d..e79d97caf07 100644 --- a/conans/client/require_resolver.py +++ b/conans/client/require_resolver.py @@ -1,6 +1,5 @@ from conans.model.ref import ConanFileReference from conans.errors import ConanException -import re def satisfying(list_versions, versionexpr, output): @@ -24,7 +23,6 @@ def satisfying(list_versions, versionexpr, output): class RequireResolver(object): - expr_pattern = re.compile("") def __init__(self, output, local_search, remote_search): self._output = output diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py index 6b174d83b11..f7311a812c4 100644 --- a/conans/test/model/version_ranges_test.py +++ b/conans/test/model/version_ranges_test.py @@ -17,6 +17,17 @@ class BasicMaxVersionTest(unittest.TestCase): + def prereleases_versions_test(self): + output = TestBufferConanOutput() + result = satisfying(["1.1.1", "1.1.11", "1.1.21", "1.1.111"], "", output) + self.assertEqual(result, "1.1.111") + # prereleases are ordered + result = satisfying(["1.1.1-1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) + self.assertEqual(result, "1.1.1-111") + # a glitch in the semver library, in theory the result might be 1.1.1 + result = satisfying(["1.1.1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) + self.assertEqual(result, "1.1.1-111") + def basic_test(self): output = TestBufferConanOutput() result = satisfying(["1.1", "1.2", "1.3", "2.1"], "", output) From 0e3d7205c9c78343f6bd9c647100b3dc988cad5e Mon Sep 17 00:00:00 2001 From: James Date: Wed, 13 Sep 2017 09:24:15 +0200 Subject: [PATCH 24/39] generators refactor (#1750) * Implement vsprops creation. * Add a simple testcase for the nbew visual_studio_legacy generator. * minor style and some boy-scout rule * merged develop and fixed test * updated contributors * working on generators refactor * generator refactor * fixed contributors --- conans/client/generators/__init__.py | 60 +++++++++++++++++--------- conans/client/generators/virtualenv.py | 1 - conans/client/loader_parse.py | 6 +-- conans/model/__init__.py | 4 +- conans/model/conan_generator.py | 28 ------------ 5 files changed, 43 insertions(+), 56 deletions(-) diff --git a/conans/client/generators/__init__.py b/conans/client/generators/__init__.py index 1abcabb296e..15f4404b407 100644 --- a/conans/client/generators/__init__.py +++ b/conans/client/generators/__init__.py @@ -1,8 +1,9 @@ -from conans.client.generators.virtualrunenv import VirtualRunEnvGenerator +from os.path import join + from conans.errors import ConanException -from conans.model import registered_generators from conans.util.files import save, normalize -from os.path import join + +from .virtualrunenv import VirtualRunEnvGenerator from .text import TXTGenerator from .gcc import GCCGenerator from .cmake import CMakeGenerator @@ -19,26 +20,43 @@ from .virtualbuildenv import VirtualBuildEnvGenerator -def _save_generator(name, klass): - if name not in registered_generators: - registered_generators.add(name, klass) +class _GeneratorManager(object): + def __init__(self): + self._generators = {} + + def add(self, name, generator_class): + if name not in self._generators: + self._generators[name] = generator_class + + @property + def available(self): + return list(self._generators.keys()) + + def __contains__(self, name): + return name in self._generators + + def __getitem__(self, key): + return self._generators[key] + + +registered_generators = _GeneratorManager() -_save_generator("txt", TXTGenerator) -_save_generator("gcc", GCCGenerator) -_save_generator("cmake", CMakeGenerator) -_save_generator("cmake_multi", CMakeMultiGenerator) -_save_generator("qmake", QmakeGenerator) -_save_generator("qbs", QbsGenerator) -_save_generator("scons", SConsGenerator) -_save_generator("visual_studio", VisualStudioGenerator) -_save_generator("visual_studio_legacy", VisualStudioLegacyGenerator) -_save_generator("xcode", XCodeGenerator) -_save_generator("ycm", YouCompleteMeGenerator) -_save_generator("virtualenv", VirtualEnvGenerator) -_save_generator("env", ConanEnvGenerator) -_save_generator("virtualbuildenv", VirtualBuildEnvGenerator) -_save_generator("virtualrunenv", VirtualRunEnvGenerator) +registered_generators.add("txt", TXTGenerator) +registered_generators.add("gcc", GCCGenerator) +registered_generators.add("cmake", CMakeGenerator) +registered_generators.add("cmake_multi", CMakeMultiGenerator) +registered_generators.add("qmake", QmakeGenerator) +registered_generators.add("qbs", QbsGenerator) +registered_generators.add("scons", SConsGenerator) +registered_generators.add("visual_studio", VisualStudioGenerator) +registered_generators.add("visual_studio_legacy", VisualStudioLegacyGenerator) +registered_generators.add("xcode", XCodeGenerator) +registered_generators.add("ycm", YouCompleteMeGenerator) +registered_generators.add("virtualenv", VirtualEnvGenerator) +registered_generators.add("env", ConanEnvGenerator) +registered_generators.add("virtualbuildenv", VirtualBuildEnvGenerator) +registered_generators.add("virtualrunenv", VirtualRunEnvGenerator) def write_generators(conanfile, path, output): diff --git a/conans/client/generators/virtualenv.py b/conans/client/generators/virtualenv.py index 14486c5bad5..35cb37378a0 100644 --- a/conans/client/generators/virtualenv.py +++ b/conans/client/generators/virtualenv.py @@ -1,6 +1,5 @@ import os import platform - from conans.model import Generator diff --git a/conans/client/loader_parse.py b/conans/client/loader_parse.py index e810b642901..457215e63fc 100644 --- a/conans/client/loader_parse.py +++ b/conans/client/loader_parse.py @@ -4,13 +4,13 @@ import sys import uuid -from conans.client.generators import _save_generator from conans.errors import ConanException, NotFoundException from conans.model.conan_file import ConanFile -from conans.model.conan_generator import Generator from conans.util.config_parser import ConfigParser from conans.util.files import rmdir from conans.tools import chdir +from conans.client.generators import registered_generators +from conans.model import Generator def load_conanfile_class(conanfile_path): @@ -40,7 +40,7 @@ class defining the Recipe, but also process possible existing generators raise ConanException("More than 1 conanfile in the file") if (inspect.isclass(attr) and issubclass(attr, Generator) and attr != Generator and attr.__dict__["__module__"] == filename): - _save_generator(attr.__name__, attr) + registered_generators.add(attr.__name__, attr) if result is None: raise ConanException("No subclass of ConanFile") diff --git a/conans/model/__init__.py b/conans/model/__init__.py index 0e6f2ac3e9a..b7a948e27df 100644 --- a/conans/model/__init__.py +++ b/conans/model/__init__.py @@ -1,3 +1 @@ -from .conan_generator import GeneratorManager, Generator - -registered_generators = GeneratorManager() +from .conan_generator import Generator diff --git a/conans/model/conan_generator.py b/conans/model/conan_generator.py index 26ec978610f..1a5c8195134 100644 --- a/conans/model/conan_generator.py +++ b/conans/model/conan_generator.py @@ -1,4 +1,3 @@ -from conans.errors import ConanException from abc import ABCMeta, abstractproperty @@ -44,30 +43,3 @@ def content(self): @abstractproperty def filename(self): raise NotImplementedError() - - -class GeneratorManager(object): - def __init__(self): - self._known_generators = {} - - def add(self, name, generator_class): - if name in self._known_generators: - raise ConanException("") - elif not issubclass(generator_class, Generator): - raise ConanException("") - else: - self._known_generators[name] = generator_class - - def remove(self, name): - if name in self._known_generators: - del self._known_generators[name] - - @property - def available(self): - return list(self._known_generators.keys()) - - def __contains__(self, key): - return key in self._known_generators - - def __getitem__(self, key): - return self._known_generators[key] From 8b16ebfed3bd13f5a71982d8495f5b214d616c61 Mon Sep 17 00:00:00 2001 From: Jonathan Newnham Date: Wed, 13 Sep 2017 17:54:13 +1000 Subject: [PATCH 25/39] Add include directories for the MIDL compiler (#1754) visual_studio generator: Add include directories for the MIDL compiler, so that it can find and use *.idl files (use the same directories as normal header includes) --- conans/client/generators/visualstudio.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conans/client/generators/visualstudio.py b/conans/client/generators/visualstudio.py index 35d576da5d2..ceaa0bef418 100644 --- a/conans/client/generators/visualstudio.py +++ b/conans/client/generators/visualstudio.py @@ -29,6 +29,9 @@ class VisualStudioGenerator(Generator): {libs}%(AdditionalDependencies) {linker_flags} %(AdditionalOptions) + + {include_dirs}%(AdditionalIncludeDirectories) + ''' From 267404ca563f0d3c58b2885847c0ede2aa523a38 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 Sep 2017 08:58:41 +0200 Subject: [PATCH 26/39] Feature/version range new test (#1756) * new version-range test * updated to node-semver 0.2.0 and fixed tests * fixed broken test --- conans/client/require_resolver.py | 2 -- conans/requirements.txt | 2 +- conans/test/model/version_ranges_test.py | 17 ++++++++--------- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/conans/client/require_resolver.py b/conans/client/require_resolver.py index e79d97caf07..d0cf3900ad0 100644 --- a/conans/client/require_resolver.py +++ b/conans/client/require_resolver.py @@ -13,8 +13,6 @@ def satisfying(list_versions, versionexpr, output): for v in list_versions: try: ver = SemVer(v, loose=True) - if not ver.prerelease: # Hack to allow version "2.1" match expr "<=2.1" - ver.prerelease = [0] candidates[ver] = v except (ValueError, AttributeError): output.warn("Version '%s' is not semver, cannot be compared with a range" % str(v)) diff --git a/conans/requirements.txt b/conans/requirements.txt index 62427ce9c52..7a3148eb7ca 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -5,7 +5,7 @@ PyYAML>=3.11, <3.13.0 patch==1.16 fasteners>=0.14.1 six>=1.10.0 -node-semver==0.1.1 +node-semver==0.2.0 distro>=1.0.2, <1.1.0 pylint>=1.6.5, <=1.8.0 future==0.16.0 diff --git a/conans/test/model/version_ranges_test.py b/conans/test/model/version_ranges_test.py index f7311a812c4..e710338faad 100644 --- a/conans/test/model/version_ranges_test.py +++ b/conans/test/model/version_ranges_test.py @@ -22,11 +22,10 @@ def prereleases_versions_test(self): result = satisfying(["1.1.1", "1.1.11", "1.1.21", "1.1.111"], "", output) self.assertEqual(result, "1.1.111") # prereleases are ordered - result = satisfying(["1.1.1-1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) - self.assertEqual(result, "1.1.1-111") - # a glitch in the semver library, in theory the result might be 1.1.1 + result = satisfying(["1.1.1-a.1", "1.1.1-a.11", "1.1.1-a.111", "1.1.1-a.21"], "~1.1.1-a", output) + self.assertEqual(result, "1.1.1-a.111") result = satisfying(["1.1.1", "1.1.1-11", "1.1.1-111", "1.1.1-21"], "", output) - self.assertEqual(result, "1.1.1-111") + self.assertEqual(result, "1.1.1") def basic_test(self): output = TestBufferConanOutput() @@ -53,7 +52,7 @@ def basic_test(self): result = satisfying(["1.6.1"], ">1.5.0,<1.6.8", output) self.assertEqual(result, "1.6.1") result = satisfying(["1.1.1", "1.1.2", "1.2", "1.2.1", "1.3", "2.1"], "<=1.2", output) - self.assertEqual(result, "1.2") + self.assertEqual(result, "1.2.1") result = satisfying(["1.1.1", "1.1.2", "1.2", "1.2.1", "1.3", "2.1"], "<1.3", output) self.assertEqual(result, "1.2.1") result = satisfying(["1.a.1", "master", "X.2", "1.2.1", "1.3", "2.1"], "1.3", output) @@ -67,9 +66,9 @@ def basic_test(self): result = satisfying(["1.3.0", "1.3.1"], "<1.3", output) self.assertEqual(result, None) result = satisfying(["1.3", "1.3.1"], "<=1.3", output) - self.assertEqual(result, "1.3") + self.assertEqual(result, "1.3.1") result = satisfying(["1.3.0", "1.3.1"], "<=1.3", output) - self.assertEqual(result, "1.3.0") + self.assertEqual(result, "1.3.1") # >2 means >=3.0.0-0 result = satisfying(["2.1"], ">2", output) self.assertEqual(result, None) @@ -224,9 +223,9 @@ def test_remote_basic(self): ('("Say/1.1@memsharded/testing", "override")', "1.1", True, False), ('("Say/0.2@memsharded/testing", "override")', "0.2", True, True), # ranges - ('"Say/[<=1.2]@memsharded/testing"', "1.1.2", False, False), + ('"Say/[<=1.2]@memsharded/testing"', "1.2.1", False, False), ('"Say/[>=0.2,<=1.0]@memsharded/testing"', "0.3", False, True), - ('("Say/[<=1.2]@memsharded/testing", "override")', "1.1.2", True, False), + ('("Say/[<=1.2]@memsharded/testing", "override")', "1.2.1", True, False), ('("Say/[>=0.2,<=1.0]@memsharded/testing", "override")', "0.3", True, True), ]) def transitive_test(self, version_range, solution, override, valid): From 68a2ed416d12b80fdc2b40d13b96635c34b30a8c Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 Sep 2017 08:58:50 +0200 Subject: [PATCH 27/39] proposed dirty abstraction (#1752) * proposed dirty abstraction * fixed dirty filename --- conans/client/export.py | 7 +++---- conans/client/installer.py | 3 +-- conans/client/manager.py | 3 +-- conans/client/source.py | 12 ++++++------ conans/paths.py | 5 ----- conans/util/files.py | 17 +++++++++++++++++ 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/conans/client/export.py b/conans/client/export.py index 9e79a3795f1..29f9e7db6d2 100644 --- a/conans/client/export.py +++ b/conans/client/export.py @@ -4,7 +4,7 @@ import shutil import os -from conans.util.files import save, load, rmdir +from conans.util.files import save, load, rmdir, is_dirty, set_dirty from conans.paths import CONAN_MANIFEST, CONANFILE from conans.errors import ConanException from conans.model.manifest import FileTreeManifest @@ -73,9 +73,8 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so save(os.path.join(destination_folder, CONAN_MANIFEST), str(digest)) source = paths.source(conan_ref, conanfile.short_paths) - dirty_file_path = paths.dirty_sources_file(conan_ref) remove = False - if os.path.exists(dirty_file_path): + if is_dirty(source): output.info("Source folder is dirty, forcing removal") remove = True elif modified_recipe and not keep_source and os.path.exists(source): @@ -91,7 +90,7 @@ def export_conanfile(output, paths, conanfile, origin_folder, conan_ref, keep_so output.error("Unable to delete source folder. " "Will be marked as dirty for deletion") output.warn(str(e)) - save(dirty_file_path, "") + set_dirty(source) def _init_export_folder(destination_folder, destination_src_folder): diff --git a/conans/client/installer.py b/conans/client/installer.py index 671c2d5c00b..26acc3302d5 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -109,7 +109,6 @@ def _build_package(self): export_folder = self._client_cache.export(self._conan_ref) export_source_folder = self._client_cache.export_sources(self._conan_ref, self._conan_file.short_paths) - dirty_file_path = self._client_cache.dirty_sources_file(self._conan_ref) try: rmdir(self.build_folder) @@ -119,7 +118,7 @@ def _build_package(self): "Close any app using it, and retry" % str(e)) self._out.info('Building your package in %s' % self.build_folder) - config_source(export_folder, export_source_folder, src_folder, dirty_file_path, + config_source(export_folder, export_source_folder, src_folder, self._conan_file, self._out) self._out.info('Copying sources to build folder') diff --git a/conans/client/manager.py b/conans/client/manager.py index 1af1d49dd73..3adde12c21c 100644 --- a/conans/client/manager.py +++ b/conans/client/manager.py @@ -437,11 +437,10 @@ def source(self, current_path, reference, force): conanfile = self.load_consumer_conanfile(conanfile_path, current_path, output, reference=reference, deps_cpp_info_required=None) - dirty_file_path = self._client_cache.dirty_sources_file(reference) src_folder = self._client_cache.source(reference, conanfile.short_paths) export_folder = self._client_cache.export(reference) export_src_folder = self._client_cache.export_sources(reference, conanfile.short_paths) - config_source(export_folder, export_src_folder, src_folder, dirty_file_path, conanfile, output, force) + config_source(export_folder, export_src_folder, src_folder, conanfile, output, force) def imports_undo(self, current_path): undo_imports(current_path, self._user_io.out) diff --git a/conans/client/source.py b/conans/client/source.py index d6c7aa987d1..b8ef83a2241 100644 --- a/conans/client/source.py +++ b/conans/client/source.py @@ -7,7 +7,7 @@ from conans.errors import ConanException, conanfile_exception_formatter, \ ConanExceptionInUserConanfileMethod from conans.paths import EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE, CONAN_MANIFEST -from conans.util.files import rmdir, save +from conans.util.files import rmdir, set_dirty, is_dirty, clean_dirty def merge_directories(src, dst): @@ -21,7 +21,7 @@ def merge_directories(src, dst): shutil.copy2(src_file, dst_file) -def config_source(export_folder, export_source_folder, src_folder, dirty_file_path, +def config_source(export_folder, export_source_folder, src_folder, conan_file, output, force=False): """ creates src folder and retrieve, calling source() from conanfile the necessary source code @@ -32,7 +32,7 @@ def remove_source(raise_error=True): try: rmdir(src_folder) except BaseException as e_rm: - save(dirty_file_path, "") # Creation of DIRTY flag + set_dirty(src_folder) msg = str(e_rm) if six.PY2: msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1 @@ -44,7 +44,7 @@ def remove_source(raise_error=True): if force: output.warn("Forced removal of source folder") remove_source() - elif os.path.exists(dirty_file_path): + elif is_dirty(src_folder): output.warn("Trying to remove dirty source folder") remove_source() elif conan_file.build_policy_always: @@ -67,13 +67,13 @@ def remove_source(raise_error=True): except OSError: pass - save(dirty_file_path, "") # Creation of DIRTY flag + set_dirty(src_folder) os.chdir(src_folder) try: with tools.environment_append(conan_file.env): with conanfile_exception_formatter(str(conan_file), "source"): conan_file.source() - os.remove(dirty_file_path) # Everything went well, remove DIRTY flag + clean_dirty(src_folder) # Everything went well, remove DIRTY flag except Exception as e: os.chdir(export_folder) # in case source() fails (user error, typically), remove the src_folder diff --git a/conans/paths.py b/conans/paths.py index a3dd696d885..907c928273c 100644 --- a/conans/paths.py +++ b/conans/paths.py @@ -36,7 +36,6 @@ def path_shortener(x, _): CONANINFO = "conaninfo.txt" CONANENV = "conanenv.txt" SYSTEM_REQS = "system_reqs.txt" -SOURCE_DIRTY_FILE = ".conan_source_dirty" PUT_HEADERS = "artifacts.properties" PACKAGE_TGZ_NAME = "conan_package.tgz" @@ -105,10 +104,6 @@ def conan(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) return normpath(join(self._store_folder, "/".join(conan_reference))) - def dirty_sources_file(self, conan_reference): - assert isinstance(conan_reference, ConanFileReference) - return normpath(join(self.conan(conan_reference), SOURCE_DIRTY_FILE)) - def export(self, conan_reference): assert isinstance(conan_reference, ConanFileReference) return normpath(join(self.conan(conan_reference), EXPORT_FOLDER)) diff --git a/conans/util/files.py b/conans/util/files.py index ad9765e3393..3dbc734e4a4 100644 --- a/conans/util/files.py +++ b/conans/util/files.py @@ -10,6 +10,23 @@ from conans.util.log import logger import tarfile +_DIRTY_FOLDER = ".dirty" + + +def set_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + save(dirty_file, "") + + +def clean_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + os.remove(dirty_file) + + +def is_dirty(folder): + dirty_file = os.path.normpath(folder) + _DIRTY_FOLDER + return os.path.exists(dirty_file) + def decode_text(text): decoders = ["utf-8", "Windows-1252"] From 6676ea70652440c09f09e42cb83ae17f4a69af99 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 Sep 2017 08:59:11 +0200 Subject: [PATCH 28/39] use VS2015 not VS2017, and simplified slow test (#1744) * use VS2015 not VS2017, and simplified slow test * fix broken tests * restored usage of default VS * using vs2015 as default --- conans/model/profile.py | 2 +- conans/test/command/profile_test.py | 6 ++--- .../conan_settings_preprocessor_test.py | 4 +-- conans/test/integration/basic_build_test.py | 27 +++++++------------ conans/test/utils/tools.py | 12 ++++++--- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/conans/model/profile.py b/conans/model/profile.py index c702a861bcc..b1825947193 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -82,7 +82,7 @@ def update_settings(self, new_settings): # Example: new_settings declare a different "compiler", so invalidate the current "compiler.XXX" for name, value in new_settings.items(): if "." not in name: - if name in self.settings and self.settings[name] != new_settings[name]: + if name in self.settings and self.settings[name] != value: for cur_name, _ in self.settings.items(): if cur_name.startswith(name): del res[cur_name] diff --git a/conans/test/command/profile_test.py b/conans/test/command/profile_test.py index 8daef183494..9429c8c3fd0 100644 --- a/conans/test/command/profile_test.py +++ b/conans/test/command/profile_test.py @@ -10,12 +10,12 @@ class ProfileTest(unittest.TestCase): def empty_test(self): - client = TestClient() + client = TestClient(default_profile=False) client.run("profile list") self.assertIn("No profiles defined", client.user_io.out) def list_test(self): - client = TestClient() + client = TestClient(default_profile=False) create_profile(client.client_cache.profiles_path, "profile3") create_profile(client.client_cache.profiles_path, "profile1") create_profile(client.client_cache.profiles_path, "profile2") @@ -24,7 +24,7 @@ def list_test(self): list(str(client.user_io.out).splitlines())) def show_test(self): - client = TestClient() + client = TestClient(default_profile=False) create_profile(client.client_cache.profiles_path, "profile1", settings={"os": "Windows"}, options=[("MyOption", "32")]) create_profile(client.client_cache.profiles_path, "profile2", scopes={"test": True}) diff --git a/conans/test/functional/conan_settings_preprocessor_test.py b/conans/test/functional/conan_settings_preprocessor_test.py index 70bf4bd0dc6..f36452655f1 100644 --- a/conans/test/functional/conan_settings_preprocessor_test.py +++ b/conans/test/functional/conan_settings_preprocessor_test.py @@ -8,7 +8,7 @@ class ConanSettingsPreprocessorTest(unittest.TestCase): def setUp(self): - self.client = TestClient() + self.client = TestClient(default_profile=False) self.conanfile = ''' from conans import ConanFile @@ -52,4 +52,4 @@ def test_runtime_not_present_ok(self): # Now install, the preprocessor shouldn't fail nor do anything self.client.run("install Hello0/0.1@lasote/channel --build missing") self.assertNotIn("Setting 'compiler.runtime' not declared, automatically", - self.client.user_io.out) \ No newline at end of file + self.client.user_io.out) diff --git a/conans/test/integration/basic_build_test.py b/conans/test/integration/basic_build_test.py index b4aec86f341..90ab4b70073 100644 --- a/conans/test/integration/basic_build_test.py +++ b/conans/test/integration/basic_build_test.py @@ -37,23 +37,17 @@ def _build(self, cmd, static, pure_c, use_cmake, lang): self.assertFalse(conan_info.full_options.static) def build_cmake_test(self): - for pure_c in (False, True): - for cmd, lang, static in [("install", 0, True), - ("install -o language=1", 1, True), - ("install -o language=1 -o static=False", 1, False), - ("install -o static=False", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=True, lang=lang) + for cmd, lang, static, pure_c in [("install", 0, True, True), + ("install -o language=1 -o static=False", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=True, lang=lang) def build_default_test(self): "build default (gcc in nix, VS in win)" if platform.system() == "SunOS": return # If is using sun-cc the gcc generator doesn't work - for pure_c in (False, True): - for cmd, lang, static in [("install -g txt", 0, True), - ("install -o language=1 -g txt", 1, True), - ("install -o language=1 -o static=False -g txt", 1, False), - ("install -o static=False -g txt", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=False, lang=lang) + for cmd, lang, static, pure_c in [("install -g txt", 0, True, True), + ("install -o language=1 -o static=False -g txt", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=False, lang=lang) def build_mingw_test(self): if platform.system() != "Windows": @@ -63,9 +57,6 @@ def build_mingw_test(self): logger.error("This platform does not support G++ command") return install = "install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9" - for pure_c in (False, True): - for cmd, lang, static in [(install, 0, True), - (install + " -o language=1", 1, True), - (install + " -o language=1 -o static=False", 1, False), - (install + " -o static=False", 0, False)]: - self._build(cmd, static, pure_c, use_cmake=False, lang=lang) + for cmd, lang, static, pure_c in [(install, 0, True, True), + (install + " -o language=1 -o static=False", 1, False, False)]: + self._build(cmd, static, pure_c, use_cmake=False, lang=lang) diff --git a/conans/test/utils/tools.py b/conans/test/utils/tools.py index 34141517df3..b5261a3de9d 100644 --- a/conans/test/utils/tools.py +++ b/conans/test/utils/tools.py @@ -35,8 +35,7 @@ TestServerLauncher) from conans.test.utils.runner import TestRunner from conans.test.utils.test_files import temp_folder -from conans.util.env_reader import get_env -from conans.util.files import save_files, load, save +from conans.util.files import save_files, save from conans.util.log import logger @@ -295,7 +294,7 @@ class TestClient(object): def __init__(self, base_folder=None, current_folder=None, servers=None, users=None, client_version=CLIENT_VERSION, min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION, - requester_class=None, runner=None, path_with_spaces=True): + requester_class=None, runner=None, path_with_spaces=True, default_profile=True): """ storage_folder: Local storage path current_folder: Current execution folder @@ -335,6 +334,13 @@ def __init__(self, base_folder=None, current_folder=None, logger.debug("Client storage = %s" % self.storage_folder) self.current_folder = current_folder or temp_folder(path_with_spaces) + # Enforcing VS 2015, even if VS2017 is auto detected + if default_profile: + profile = self.client_cache.default_profile + if profile.settings.get("compiler.version") == "15": + profile.settings["compiler.version"] = "14" + save(self.client_cache.default_profile_path, profile.dumps()) + @property def paths(self): return self.client_cache From 2f858de0b67411cc337b2964ba862361ee3ebfdf Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Fri, 15 Sep 2017 08:29:43 +0200 Subject: [PATCH 29/39] nosetests multiprocess CI (#1759) * 4 Processes * catch rm problem * Try with 2 processes * Restored 4 and removed long log for win * Recovered debug --- .ci/appveyor/test.bat | 2 +- .ci/travis/run.sh | 2 +- conans/test/server/utils/server_launcher.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.ci/appveyor/test.bat b/.ci/appveyor/test.bat index f72fb740f75..d61badbf253 100644 --- a/.ci/appveyor/test.bat +++ b/.ci/appveyor/test.bat @@ -1 +1 @@ -nosetests --with-coverage --verbosity=2 conans.test +nosetests --with-coverage --verbosity=2 conans.test --processes=4 --process-timeout=1000 diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 3d55b03c915..2c6c337c163 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -10,4 +10,4 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv activate conan fi -nosetests --with-coverage conans.test --verbosity=2 +nosetests --with-coverage conans.test --verbosity=2 --processes=4 --process-timeout=1000 diff --git a/conans/test/server/utils/server_launcher.py b/conans/test/server/utils/server_launcher.py index cc180212390..7cbe9ee2dad 100644 --- a/conans/test/server/utils/server_launcher.py +++ b/conans/test/server/utils/server_launcher.py @@ -117,7 +117,10 @@ def stop(self): def clean(self): if os.path.exists(self.storage_folder): - shutil.rmtree(self.storage_folder) + try: + shutil.rmtree(self.storage_folder) + except: + print("Can't clean the test server data, probably a server process is still opened") if __name__ == "__main__": From f4fef4e9aa592d8cffa4c141fc904e039c958177 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Sat, 16 Sep 2017 08:25:36 +0200 Subject: [PATCH 30/39] Tools split + Moved collect libs (#1761) * Moved collect libs * removed print * Fixed test * Fixed test --- conans/client/tools/__init__.py | 13 + conans/client/tools/env.py | 40 + conans/client/tools/files.py | 221 ++++++ conans/client/tools/net.py | 48 ++ conans/client/tools/oss.py | 233 ++++++ conans/client/tools/system_pm.py | 159 ++++ conans/client/tools/win.py | 150 ++++ conans/model/conan_file.py | 18 +- conans/test/integration/multi_build_test.py | 1 - conans/test/util/output_test.py | 7 +- conans/test/utils/cpp_test_files.py | 3 +- conans/tools.py | 817 +------------------- 12 files changed, 892 insertions(+), 818 deletions(-) create mode 100644 conans/client/tools/__init__.py create mode 100644 conans/client/tools/env.py create mode 100644 conans/client/tools/files.py create mode 100644 conans/client/tools/net.py create mode 100644 conans/client/tools/oss.py create mode 100644 conans/client/tools/system_pm.py create mode 100644 conans/client/tools/win.py diff --git a/conans/client/tools/__init__.py b/conans/client/tools/__init__.py new file mode 100644 index 00000000000..a36eae84c0f --- /dev/null +++ b/conans/client/tools/__init__.py @@ -0,0 +1,13 @@ + +# noinspection PyUnresolvedReferences +from .env import * +# noinspection PyUnresolvedReferences +from .files import * +# noinspection PyUnresolvedReferences +from .net import * +# noinspection PyUnresolvedReferences +from .oss import * +# noinspection PyUnresolvedReferences +from .system_pm import * +# noinspection PyUnresolvedReferences +from .win import * diff --git a/conans/client/tools/env.py b/conans/client/tools/env.py new file mode 100644 index 00000000000..2c60854efd9 --- /dev/null +++ b/conans/client/tools/env.py @@ -0,0 +1,40 @@ +import sys +from contextlib import contextmanager + +import os + + +@contextmanager +def pythonpath(conanfile): + old_path = sys.path[:] + python_path = conanfile.env.get("PYTHONPATH", None) + if python_path: + if isinstance(python_path, list): + sys.path.extend(python_path) + else: + sys.path.append(python_path) + + yield + sys.path = old_path + + +@contextmanager +def environment_append(env_vars): + """ + :param env_vars: List of simple environment vars. {name: value, name2: value2} => e.j: MYVAR=1 + The values can also be lists of appendable environment vars. {name: [value, value2]} + => e.j. PATH=/path/1:/path/2 + :return: None + """ + old_env = dict(os.environ) + for name, value in env_vars.items(): + if isinstance(value, list): + env_vars[name] = os.pathsep.join(value) + if name in old_env: + env_vars[name] += os.pathsep + old_env[name] + os.environ.update(env_vars) + try: + yield + finally: + os.environ.clear() + os.environ.update(old_env) diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py new file mode 100644 index 00000000000..a52fe94486c --- /dev/null +++ b/conans/client/tools/files.py @@ -0,0 +1,221 @@ +import platform +from contextlib import contextmanager + +import logging + +import re + +import os +import sys + +from conans.client.output import ConanOutput +from conans.errors import ConanException +from conans.util.files import load, save, _generic_algorithm_sum +from patch import fromfile, fromstring + + +_global_output = None + + +@contextmanager +def chdir(newdir): + old_path = os.getcwd() + os.chdir(newdir) + try: + yield + finally: + os.chdir(old_path) + + +def human_size(size_bytes): + """ + format a size in bytes into a 'human' file size, e.g. B, KB, MB, GB, TB, PB + Note that bytes will be reported in whole numbers but KB and above will have + greater precision. e.g. 43 B, 443 KB, 4.3 MB, 4.43 GB, etc + """ + + suffixes_table = [('B', 0), ('KB', 1), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)] + + num = float(size_bytes) + for suffix, precision in suffixes_table: + if num < 1024.0: + break + num /= 1024.0 + + if precision == 0: + formatted_size = "%d" % num + else: + formatted_size = str(round(num, ndigits=precision)) + + return "%s%s" % (formatted_size, suffix) + + +def unzip(filename, destination=".", keep_permissions=False): + """ + Unzip a zipped file + :param filename: Path to the zip file + :param destination: Destination folder + :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX + system, the bits could produce undefined permission schema. Use only this option if you are sure that the + zip was created correctly. + :return: + """ + if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or + filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or + filename.endswith(".tar")): + return untargz(filename, destination) + import zipfile + full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) + + if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): + def print_progress(the_size, uncomp_size): + txt_msg = "Unzipping %.0f %%" % (the_size * 100.0 / uncomp_size) + _global_output.rewrite_line(txt_msg) + else: + def print_progress(_, __): + pass + + with zipfile.ZipFile(filename, "r") as z: + uncompress_size = sum((file_.file_size for file_ in z.infolist())) + if uncompress_size > 100000: + _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) + else: + _global_output.info("Unzipping %s" % human_size(uncompress_size)) + extracted_size = 0 + if platform.system() == "Windows": + for file_ in z.infolist(): + extracted_size += file_.file_size + print_progress(extracted_size, uncompress_size) + try: + # Win path limit is 260 chars + if len(file_.filename) + len(full_path) >= 260: + raise ValueError("Filename too long") + z.extract(file_, full_path) + except Exception as e: + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) + else: # duplicated for, to avoid a platform check for each zipped file + for file_ in z.infolist(): + extracted_size += file_.file_size + print_progress(extracted_size, uncompress_size) + try: + z.extract(file_, full_path) + if keep_permissions: + # Could be dangerous if the ZIP has been created in a non nix system + # https://bugs.python.org/issue15795 + perm = file_.external_attr >> 16 & 0xFFF + os.chmod(os.path.join(full_path, file_.filename), perm) + except Exception as e: + _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) + + +def untargz(filename, destination="."): + import tarfile + with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: + tarredgzippedFile.extractall(destination) + + +def check_with_algorithm_sum(algorithm_name, file_path, signature): + real_signature = _generic_algorithm_sum(file_path, algorithm_name) + if real_signature != signature: + raise ConanException("%s signature failed for '%s' file." + " Computed signature: %s" % (algorithm_name, + os.path.basename(file_path), + real_signature)) + + +def check_sha1(file_path, signature): + check_with_algorithm_sum("sha1", file_path, signature) + + +def check_md5(file_path, signature): + check_with_algorithm_sum("md5", file_path, signature) + + +def check_sha256(file_path, signature): + check_with_algorithm_sum("sha256", file_path, signature) + + +def patch(base_path=None, patch_file=None, patch_string=None, strip=0, output=None): + """Applies a diff from file (patch_file) or string (patch_string) + in base_path directory or current dir if None""" + + class PatchLogHandler(logging.Handler): + def __init__(self): + logging.Handler.__init__(self, logging.DEBUG) + self.output = output or ConanOutput(sys.stdout, True) + self.patchname = patch_file if patch_file else "patch" + + def emit(self, record): + logstr = self.format(record) + if record.levelno == logging.WARN: + self.output.warn("%s: %s" % (self.patchname, logstr)) + else: + self.output.info("%s: %s" % (self.patchname, logstr)) + + patchlog = logging.getLogger("patch") + if patchlog: + patchlog.handlers = [] + patchlog.addHandler(PatchLogHandler()) + + if not patch_file and not patch_string: + return + if patch_file: + patchset = fromfile(patch_file) + else: + patchset = fromstring(patch_string.encode()) + + if not patchset: + raise ConanException("Failed to parse patch: %s" % (patch_file if patch_file else "string")) + + if not patchset.apply(root=base_path, strip=strip): + raise ConanException("Failed to apply patch: %s" % patch_file) + + +def replace_in_file(file_path, search, replace, strict=True): + content = load(file_path) + if -1 == content.find(search): + message = "replace_in_file didn't find pattern '%s' in '%s' file." % (search, file_path) + if strict: + raise ConanException(message) + else: + _global_output.warn(message) + content = content.replace(search, replace) + content = content.encode("utf-8") + with open(file_path, "wb") as handle: + handle.write(content) + + +def replace_prefix_in_pc_file(pc_file, new_prefix): + content = load(pc_file) + lines = [] + for line in content.splitlines(): + if line.startswith("prefix="): + lines.append('prefix=%s' % new_prefix) + else: + lines.append(line) + save(pc_file, "\n".join(lines)) + + +def unix_path(path): + """"Used to translate windows paths to MSYS unix paths like + c/users/path/to/file""" + pattern = re.compile(r'([a-z]):\\', re.IGNORECASE) + return pattern.sub('/\\1/', path).replace('\\', '/').lower() + + +def collect_libs(conanfile, folder="lib"): + if not conanfile.package_folder: + return [] + lib_folder = os.path.join(conanfile.package_folder, folder) + if not os.path.exists(lib_folder): + conanfile.output.warn("Lib folder doesn't exist, can't collect libraries") + return [] + files = os.listdir(lib_folder) + result = [] + for f in files: + name, ext = os.path.splitext(f) + if ext in (".so", ".lib", ".a", ".dylib"): + if ext != ".lib" and name.startswith("lib"): + name = name[3:] + result.append(name) + return result \ No newline at end of file diff --git a/conans/client/tools/net.py b/conans/client/tools/net.py new file mode 100644 index 00000000000..32dd3416a33 --- /dev/null +++ b/conans/client/tools/net.py @@ -0,0 +1,48 @@ +import sys + +import os +from conans.client.output import ConanOutput +from conans.client.rest.uploader_downloader import Downloader +from conans.client.tools.files import unzip +from conans.errors import ConanException + +_global_requester = None + + +def get(url): + """ high level downloader + unziper + delete temporary zip + """ + filename = os.path.basename(url) + download(url, filename) + unzip(filename) + os.unlink(filename) + + +def ftp_download(ip, filename, login='', password=''): + import ftplib + try: + ftp = ftplib.FTP(ip, login, password) + ftp.login() + filepath, filename = os.path.split(filename) + if filepath: + ftp.cwd(filepath) + with open(filename, 'wb') as f: + ftp.retrbinary('RETR ' + filename, f.write) + except Exception as e: + raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) + finally: + try: + ftp.quit() + except: + pass + + +def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): + out = out or ConanOutput(sys.stdout, True) + if verify: + # We check the certificate using a list of known verifiers + import conans.client.rest.cacert as cacert + verify = cacert.file_path + downloader = Downloader(_global_requester, out, verify=verify) + downloader.download(url, filename, retry=retry, retry_wait=retry_wait) + out.writeln("") diff --git a/conans/client/tools/oss.py b/conans/client/tools/oss.py new file mode 100644 index 00000000000..84a77c965d4 --- /dev/null +++ b/conans/client/tools/oss.py @@ -0,0 +1,233 @@ +import multiprocessing +import platform +import subprocess +import sys + +import os +from conans.model.version import Version +from conans.util.log import logger + +_global_output = None + + +def args_to_string(args): + if not args: + return "" + if sys.platform == 'win32': + return subprocess.list2cmdline(args) + else: + return " ".join("'" + arg.replace("'", r"'\''") + "'" for arg in args) + + +def cpu_count(): + try: + env_cpu_count = os.getenv("CONAN_CPU_COUNT", None) + return int(env_cpu_count) if env_cpu_count else multiprocessing.cpu_count() + except NotImplementedError: + _global_output.warn("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") + return 1 # Safe guess + + +def detected_architecture(): + # FIXME: Very weak check but not very common to run conan in other architectures + if "64" in platform.machine(): + return "x86_64" + elif "86" in platform.machine(): + return "x86" + return None + +# DETECT OS, VERSION AND DISTRIBUTIONS + + +class OSInfo(object): + """ Usage: + (os_info.is_linux) # True/False + (os_info.is_windows) # True/False + (os_info.is_macos) # True/False + (os_info.is_freebsd) # True/False + (os_info.is_solaris) # True/False + + (os_info.linux_distro) # debian, ubuntu, fedora, centos... + + (os_info.os_version) # 5.1 + (os_info.os_version_name) # Windows 7, El Capitan + + if os_info.os_version > "10.1": + pass + if os_info.os_version == "10.1.0": + pass + """ + + def __init__(self): + self.os_version = None + self.os_version_name = None + self.is_linux = platform.system() == "Linux" + self.linux_distro = None + self.is_windows = platform.system() == "Windows" + self.is_macos = platform.system() == "Darwin" + self.is_freebsd = platform.system() == "FreeBSD" + self.is_solaris = platform.system() == "SunOS" + + if self.is_linux: + import distro + self.linux_distro = distro.id() + self.os_version = Version(distro.version()) + version_name = distro.codename() + self.os_version_name = version_name if version_name != "n/a" else "" + if not self.os_version_name and self.linux_distro == "debian": + self.os_version_name = self.get_debian_version_name(self.os_version) + elif self.is_windows: + self.os_version = self.get_win_os_version() + self.os_version_name = self.get_win_version_name(self.os_version) + elif self.is_macos: + self.os_version = Version(platform.mac_ver()[0]) + self.os_version_name = self.get_osx_version_name(self.os_version) + elif self.is_freebsd: + self.os_version = self.get_freebsd_version() + self.os_version_name = "FreeBSD %s" % self.os_version + elif self.is_solaris: + self.os_version = Version(platform.release()) + self.os_version_name = self.get_solaris_version_name(self.os_version) + + @property + def with_apt(self): + return self.is_linux and self.linux_distro in \ + ("debian", "ubuntu", "knoppix", "linuxmint", "raspbian") + + @property + def with_yum(self): + return self.is_linux and self.linux_distro in \ + ("centos", "redhat", "fedora", "pidora", "scientific", + "xenserver", "amazon", "oracle", "rhel") + + @staticmethod + def get_win_os_version(): + """ + Get's the OS major and minor versions. Returns a tuple of + (OS_MAJOR, OS_MINOR). + """ + import ctypes + + class _OSVERSIONINFOEXW(ctypes.Structure): + _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), + ('dwMajorVersion', ctypes.c_ulong), + ('dwMinorVersion', ctypes.c_ulong), + ('dwBuildNumber', ctypes.c_ulong), + ('dwPlatformId', ctypes.c_ulong), + ('szCSDVersion', ctypes.c_wchar * 128), + ('wServicePackMajor', ctypes.c_ushort), + ('wServicePackMinor', ctypes.c_ushort), + ('wSuiteMask', ctypes.c_ushort), + ('wProductType', ctypes.c_byte), + ('wReserved', ctypes.c_byte)] + + os_version = _OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + return None + + return Version("%d.%d" % (os_version.dwMajorVersion, os_version.dwMinorVersion)) + + @staticmethod + def get_debian_version_name(version): + if not version: + return None + elif version.major() == "8.Y.Z": + return "jessie" + elif version.major() == "7.Y.Z": + return "wheezy" + elif version.major() == "6.Y.Z": + return "squeeze" + elif version.major() == "5.Y.Z": + return "lenny" + elif version.major() == "4.Y.Z": + return "etch" + elif version.minor() == "3.1.Z": + return "sarge" + elif version.minor() == "3.0.Z": + return "woody" + + @staticmethod + def get_win_version_name(version): + if not version: + return None + elif version.major() == "5.Y.Z": + return "Windows XP" + elif version.minor() == "6.0.Z": + return "Windows Vista" + elif version.minor() == "6.1.Z": + return "Windows 7" + elif version.minor() == "6.2.Z": + return "Windows 8" + elif version.minor() == "6.3.Z": + return "Windows 8.1" + elif version.minor() == "10.0.Z": + return "Windows 10" + + @staticmethod + def get_osx_version_name(version): + if not version: + return None + elif version.minor() == "10.12.Z": + return "Sierra" + elif version.minor() == "10.11.Z": + return "El Capitan" + elif version.minor() == "10.10.Z": + return "Yosemite" + elif version.minor() == "10.9.Z": + return "Mavericks" + elif version.minor() == "10.8.Z": + return "Mountain Lion" + elif version.minor() == "10.7.Z": + return "Lion" + elif version.minor() == "10.6.Z": + return "Snow Leopard" + elif version.minor() == "10.5.Z": + return "Leopard" + elif version.minor() == "10.4.Z": + return "Tiger" + elif version.minor() == "10.3.Z": + return "Panther" + elif version.minor() == "10.2.Z": + return "Jaguar" + elif version.minor() == "10.1.Z": + return "Puma" + elif version.minor() == "10.0.Z": + return "Cheetha" + + @staticmethod + def get_freebsd_version(): + return platform.release().split("-")[0] + + @staticmethod + def get_solaris_version_name(version): + if not version: + return None + elif version.minor() == "5.10": + return "Solaris 10" + elif version.minor() == "5.11": + return "Solaris 11" + + +def cross_building(settings, self_os=None, self_arch=None): + self_os = self_os or platform.system() + self_arch = self_arch or detected_architecture() + os_setting = settings.get_safe("os") + arch_setting = settings.get_safe("arch") + platform_os = {"Darwin": "Macos"}.get(self_os, self_os) + if self_os == os_setting and self_arch == "x86_64" and arch_setting == "x86": + return False # not really considered cross + + if os_setting and platform_os != os_setting: + return True + if arch_setting and self_arch != arch_setting: + return True + + return False + +try: + os_info = OSInfo() +except Exception as exc: + logger.error(exc) + _global_output.error("Error detecting os_info") \ No newline at end of file diff --git a/conans/client/tools/system_pm.py b/conans/client/tools/system_pm.py new file mode 100644 index 00000000000..70acc43ca69 --- /dev/null +++ b/conans/client/tools/system_pm.py @@ -0,0 +1,159 @@ +import os +from conans.client.runner import ConanRunner +from conans.client.tools.oss import OSInfo +from conans.errors import ConanException + +_global_output = None + + +class SystemPackageTool(object): + + def __init__(self, runner=None, os_info=None, tool=None): + env_sudo = os.environ.get("CONAN_SYSREQUIRES_SUDO", None) + self._sudo = (env_sudo != "False" and env_sudo != "0") + os_info = os_info or OSInfo() + self._is_up_to_date = False + self._tool = tool or self._create_tool(os_info) + self._tool._sudo_str = "sudo " if self._sudo else "" + self._tool._runner = runner or ConanRunner() + + @staticmethod + def _create_tool(os_info): + if os_info.with_apt: + return AptTool() + elif os_info.with_yum: + return YumTool() + elif os_info.is_macos: + return BrewTool() + elif os_info.is_freebsd: + return PkgTool() + elif os_info.is_solaris: + return PkgUtilTool() + else: + return NullTool() + + def update(self): + """ + Get the system package tool update command + """ + self._is_up_to_date = True + self._tool.update() + + def install(self, packages, update=True, force=False): + ''' + Get the system package tool install command. + ''' + packages = [packages] if isinstance(packages, str) else list(packages) + if not force and self._installed(packages): + return + if update and not self._is_up_to_date: + self.update() + self._install_any(packages) + + def _installed(self, packages): + for pkg in packages: + if self._tool.installed(pkg): + _global_output.info("Package already installed: %s" % pkg) + return True + return False + + def _install_any(self, packages): + if len(packages) == 1: + return self._tool.install(packages[0]) + for pkg in packages: + try: + return self._tool.install(pkg) + except ConanException: + pass + raise ConanException("Could not install any of %s" % packages) + + +class NullTool(object): + def update(self): + pass + + def install(self, package_name): + _global_output.warn("Only available for linux with apt-get or yum or OSx with brew or " + "FreeBSD with pkg or Solaris with pkgutil") + + def installed(self, package_name): + return False + + +class AptTool(object): + def update(self): + _run(self._runner, "%sapt-get update" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%sapt-get install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("dpkg -s %s" % package_name, None) + return exit_code == 0 + + +class YumTool(object): + def update(self): + _run(self._runner, "%syum check-update" % self._sudo_str, accepted_returns=[0, 100]) + + def install(self, package_name): + _run(self._runner, "%syum install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("rpm -q %s" % package_name, None) + return exit_code == 0 + + +class BrewTool(object): + def update(self): + _run(self._runner, "brew update") + + def install(self, package_name): + _run(self._runner, "brew install %s" % package_name) + + def installed(self, package_name): + exit_code = self._runner('test -n "$(brew ls --versions %s)"' % package_name, None) + return exit_code == 0 + + +class PkgTool(object): + def update(self): + _run(self._runner, "%spkg update" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%spkg install -y %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner("pkg info %s" % package_name, None) + return exit_code == 0 + + +class PkgUtilTool(object): + def update(self): + _run(self._runner, "%spkgutil --catalog" % self._sudo_str) + + def install(self, package_name): + _run(self._runner, "%spkgutil --install --yes %s" % (self._sudo_str, package_name)) + + def installed(self, package_name): + exit_code = self._runner('test -n "`pkgutil --list %s`"' % package_name, None) + return exit_code == 0 + + +class ChocolateyTool(object): + def update(self): + _run(self._runner, "choco outdated") + + def install(self, package_name): + _run(self._runner, "choco install --yes %s" % package_name) + + def installed(self, package_name): + exit_code = self._runner('choco search --local-only --exact %s | findstr /c:"1 packages installed."' % package_name, None) + return exit_code == 0 + + +def _run(runner, command, accepted_returns=None): + accepted_returns = accepted_returns or [0, ] + _global_output.info("Running: %s" % command) + if runner(command, True) not in accepted_returns: + raise ConanException("Command '%s' failed" % command) diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py new file mode 100644 index 00000000000..f15c5182435 --- /dev/null +++ b/conans/client/tools/win.py @@ -0,0 +1,150 @@ +import os +import platform + +import subprocess + +from conans.client.tools.env import environment_append +from conans.client.tools.files import unix_path +from conans.errors import ConanException + +_global_output = None + + +def msvc_build_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, + arch=None): + """ Do both: set the environment variables and call the .sln build + """ + vcvars = vcvars_command(settings) + build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch) + command = "%s && %s" % (vcvars, build) + return command + + +def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, + arch=None): + """ + Use example: + build_command = build_sln_command(self.settings, "myfile.sln", targets=["SDL2_image"]) + command = "%s && %s" % (tools.vcvars_command(self.settings), build_command) + self.run(command) + """ + targets = targets or [] + command = "devenv %s /upgrade && " % sln_path if upgrade_project else "" + build_type = build_type or settings.build_type + arch = arch or settings.arch + if not build_type: + raise ConanException("Cannot build_sln_command, build_type not defined") + if not arch: + raise ConanException("Cannot build_sln_command, arch not defined") + command += "msbuild %s /p:Configuration=%s" % (sln_path, build_type) + arch = str(arch) + if arch in ["x86_64", "x86"]: + command += ' /p:Platform=' + command += '"x64"' if arch == "x86_64" else '"x86"' + elif "ARM" in arch.upper(): + command += ' /p:Platform="ARM"' + + if targets: + command += " /target:%s" % ";".join(targets) + return command + + +def vs_installation_path(version): + if not hasattr(vs_installation_path, "_cached"): + vs_installation_path._cached = dict() + + if version not in vs_installation_path._cached: + vs_path = None + program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles")) + if program_files: + vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer", + "vswhere.exe") + if os.path.isfile(vswhere_path): + version_range = "[%d.0, %d.0)" % (int(version), int(version) + 1) + try: + output = subprocess.check_output([vswhere_path, "-version", version_range, + "-legacy", "-property", "installationPath"]) + vs_path = output.decode().strip() + _global_output.info("vswhere detected VS %s in %s" % (version, vs_path)) + except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e: + _global_output.error("vswhere error: %s" % str(e)) + + # Remember to cache result + vs_installation_path._cached[version] = vs_path + + return vs_installation_path._cached[version] + + +def vcvars_command(settings): + arch_setting = settings.get_safe("arch") + compiler_version = settings.get_safe("compiler.version") + if not compiler_version: + raise ConanException("compiler.version setting required for vcvars not defined") + + param = "x86" if arch_setting == "x86" else "amd64" + existing_version = os.environ.get("VisualStudioVersion") + if existing_version: + command = "echo Conan:vcvars already set" + existing_version = existing_version.split(".")[0] + if existing_version != compiler_version: + raise ConanException("Error, Visual environment already set to %s\n" + "Current settings visual version: %s" + % (existing_version, compiler_version)) + else: + env_var = "vs%s0comntools" % compiler_version + + if env_var == 'vs150comntools': + vs_path = os.getenv(env_var) + if not vs_path: # Try to locate with vswhere + vs_root = vs_installation_path("15") + if vs_root: + vs_path = os.path.join(vs_root, "Common7", "Tools") + else: + raise ConanException("VS2017 '%s' variable not defined, " + "and vswhere didn't find it" % env_var) + vcvars_path = os.path.join(vs_path, "../../VC/Auxiliary/Build/vcvarsall.bat") + command = ('set "VSCMD_START_DIR=%%CD%%" && ' + 'call "%s" %s' % (vcvars_path, param)) + else: + try: + vs_path = os.environ[env_var] + except KeyError: + raise ConanException("VS '%s' variable not defined. Please install VS" % env_var) + vcvars_path = os.path.join(vs_path, "../../VC/vcvarsall.bat") + command = ('call "%s" %s' % (vcvars_path, param)) + + return command + + +def escape_windows_cmd(command): + """ To use in a regular windows cmd.exe + 1. Adds escapes so the argument can be unpacked by CommandLineToArgvW() + 2. Adds escapes for cmd.exe so the argument survives cmd.exe's substitutions. + + Useful to escape commands to be executed in a windows bash (msys2, cygwin etc) + """ + quoted_arg = subprocess.list2cmdline([command]) + return "".join(["^%s" % arg if arg in r'()%!^"<>&|' else arg for arg in quoted_arg]) + + +def run_in_windows_bash(conanfile, bashcmd, cwd=None): + """ Will run a unix command inside the msys2 environment + It requires to have MSYS2 in the path and MinGW + """ + if platform.system() != "Windows": + raise ConanException("Command only for Windows operating system") + # This needs to be set so that msys2 bash profile will set up the environment correctly. + try: + arch = conanfile.settings.arch # Maybe arch doesn't exist + except: + arch = None + env_vars = {"MSYSTEM": "MINGW32" if arch == "x86" else "MINGW64", + "MSYS2_PATH_TYPE": "inherit"} + with environment_append(env_vars): + curdir = unix_path(cwd or os.path.abspath(os.path.curdir)) + # Needed to change to that dir inside the bash shell + to_run = 'cd "%s" && %s ' % (curdir, bashcmd) + custom_bash_path = os.getenv("CONAN_BASH_PATH", "bash") + wincmd = '%s --login -c %s' % (custom_bash_path, escape_windows_cmd(to_run)) + conanfile.output.info('run_in_windows_bash: %s' % wincmd) + conanfile.run(wincmd) diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index c4665a182ce..57cffa8e2b1 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -158,21 +158,9 @@ def user(self): return self._user def collect_libs(self, folder="lib"): - if not self.package_folder: - return [] - lib_folder = os.path.join(self.package_folder, folder) - if not os.path.exists(lib_folder): - self.output.warn("Lib folder doesn't exist, can't collect libraries") - return [] - files = os.listdir(lib_folder) - result = [] - for f in files: - name, ext = os.path.splitext(f) - if ext in (".so", ".lib", ".a", ".dylib"): - if ext != ".lib" and name.startswith("lib"): - name = name[3:] - result.append(name) - return result + self.output.warn("Use 'self.collect_libs' is deprecated, " + "use tools.collect_libs(self) instead") + return tools.collect_libs(self, folder=folder) @property def scope(self): diff --git a/conans/test/integration/multi_build_test.py b/conans/test/integration/multi_build_test.py index 110eca40876..cc1c18eb1d1 100644 --- a/conans/test/integration/multi_build_test.py +++ b/conans/test/integration/multi_build_test.py @@ -23,7 +23,6 @@ def collect_libs_test(self): self.assertEquals(len(package_ids), 1) # Reuse them - conan_reference = ConanFileReference.loads("Hello1/0.2@lasote/stable") files3 = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], collect_libs=True) diff --git a/conans/test/util/output_test.py b/conans/test/util/output_test.py index eba9e2afb97..c7bc27f5ca4 100644 --- a/conans/test/util/output_test.py +++ b/conans/test/util/output_test.py @@ -65,10 +65,13 @@ def unzip_output_test(self): new_out = StringIO() old_out = sys.stdout try: - tools._global_output = ConanOutput(new_out) + import requests + import conans + + conans.tools.set_global_instances(ConanOutput(new_out), requests) tools.unzip(zip_path, output_dir) finally: - sys.stdout = old_out + conans.tools.set_global_instances(ConanOutput(old_out), requests) output = new_out.getvalue() self.assertRegexpMatches(output, "Unzipping [\d]+B") diff --git a/conans/test/utils/cpp_test_files.py b/conans/test/utils/cpp_test_files.py index ee15cfc9f5d..8daeba08f77 100644 --- a/conans/test/utils/cpp_test_files.py +++ b/conans/test/utils/cpp_test_files.py @@ -365,6 +365,7 @@ def cpp_hello_conan_files(name="Hello", version="0.1", deps=None, language=0, st if not config: conanfile = conanfile.replace("config(", "config2(") if collect_libs: - conanfile = conanfile.replace('["hello%s"]' % name, "self.collect_libs()") + conanfile = "from conans import tools\n" + conanfile.replace('["hello%s"]' % name, + "tools.collect_libs(self)") base_files[CONANFILE] = conanfile return base_files diff --git a/conans/tools.py b/conans/tools.py index 3c014f61272..45d360092f8 100644 --- a/conans/tools.py +++ b/conans/tools.py @@ -1,809 +1,28 @@ -""" ConanFile user tools, as download, etc -""" +""" ConanFile user tools, as download, etc""" from __future__ import print_function -import logging -import multiprocessing -import os -import platform -import re -import subprocess -import sys - - -from contextlib import contextmanager - import requests -from patch import fromfile, fromstring - +from conans.client.tools import * from conans.client.output import ConanOutput -from conans.client.rest.uploader_downloader import Downloader -from conans.client.runner import ConanRunner -from conans.errors import ConanException -from conans.model.version import Version # noinspection PyUnresolvedReferences -from conans.util.files import _generic_algorithm_sum, load, save, sha256sum, \ - sha1sum, md5sum, md5, touch, relative_dirs, rmdir, mkdir -from conans.util.log import logger - -# Default values -_global_requester = requests -_global_output = ConanOutput(sys.stdout) - - -def unix_path(path): - """"Used to translate windows paths to MSYS unix paths like - c/users/path/to/file""" - pattern = re.compile(r'([a-z]):\\', re.IGNORECASE) - return pattern.sub('/\\1/', path).replace('\\', '/').lower() - - -def escape_windows_cmd(command): - """ To use in a regular windows cmd.exe - 1. Adds escapes so the argument can be unpacked by CommandLineToArgvW() - 2. Adds escapes for cmd.exe so the argument survives cmd.exe's substitutions. - - Useful to escape commands to be executed in a windows bash (msys2, cygwin etc) - """ - quoted_arg = subprocess.list2cmdline([command]) - return "".join(["^%s" % arg if arg in r'()%!^"<>&|' else arg for arg in quoted_arg]) - - -def run_in_windows_bash(conanfile, bashcmd, cwd=None): - """ Will run a unix command inside the msys2 environment - It requires to have MSYS2 in the path and MinGW - """ - if platform.system() != "Windows": - raise ConanException("Command only for Windows operating system") - # This needs to be set so that msys2 bash profile will set up the environment correctly. - try: - arch = conanfile.settings.arch # Maybe arch doesn't exist - except: - arch = None - env_vars = {"MSYSTEM": "MINGW32" if arch == "x86" else "MINGW64", - "MSYS2_PATH_TYPE": "inherit"} - with environment_append(env_vars): - curdir = unix_path(cwd or os.path.abspath(os.path.curdir)) - # Needed to change to that dir inside the bash shell - to_run = 'cd "%s" && %s ' % (curdir, bashcmd) - custom_bash_path = os.getenv("CONAN_BASH_PATH", "bash") - wincmd = '%s --login -c %s' % (custom_bash_path, escape_windows_cmd(to_run)) - conanfile.output.info('run_in_windows_bash: %s' % wincmd) - conanfile.run(wincmd) - - -def args_to_string(args): - if not args: - return "" - if sys.platform == 'win32': - return subprocess.list2cmdline(args) - else: - return " ".join("'" + arg.replace("'", r"'\''") + "'" for arg in args) - - -@contextmanager -def chdir(newdir): - old_path = os.getcwd() - os.chdir(newdir) - try: - yield - finally: - os.chdir(old_path) - - -@contextmanager -def pythonpath(conanfile): - old_path = sys.path[:] - python_path = conanfile.env.get("PYTHONPATH", None) - if python_path: - if isinstance(python_path, list): - sys.path.extend(python_path) - else: - sys.path.append(python_path) - - yield - sys.path = old_path - - -@contextmanager -def environment_append(env_vars): - """ - :param env_vars: List of simple environment vars. {name: value, name2: value2} => e.j: MYVAR=1 - The values can also be lists of appendable environment vars. {name: [value, value2]} - => e.j. PATH=/path/1:/path/2 - :return: None - """ - old_env = dict(os.environ) - for name, value in env_vars.items(): - if isinstance(value, list): - env_vars[name] = os.pathsep.join(value) - if name in old_env: - env_vars[name] += os.pathsep + old_env[name] - os.environ.update(env_vars) - try: - yield - finally: - os.environ.clear() - os.environ.update(old_env) - - -def msvc_build_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None): - """ Do both: set the environment variables and call the .sln build - """ - vcvars = vcvars_command(settings) - build = build_sln_command(settings, sln_path, targets, upgrade_project, build_type, arch) - command = "%s && %s" % (vcvars, build) - return command - - -def build_sln_command(settings, sln_path, targets=None, upgrade_project=True, build_type=None, - arch=None): - """ - Use example: - build_command = build_sln_command(self.settings, "myfile.sln", targets=["SDL2_image"]) - command = "%s && %s" % (tools.vcvars_command(self.settings), build_command) - self.run(command) - """ - targets = targets or [] - command = "devenv %s /upgrade && " % sln_path if upgrade_project else "" - build_type = build_type or settings.build_type - arch = arch or settings.arch - if not build_type: - raise ConanException("Cannot build_sln_command, build_type not defined") - if not arch: - raise ConanException("Cannot build_sln_command, arch not defined") - command += "msbuild %s /p:Configuration=%s" % (sln_path, build_type) - arch = str(arch) - if arch in ["x86_64", "x86"]: - command += ' /p:Platform=' - command += '"x64"' if arch == "x86_64" else '"x86"' - elif "ARM" in arch.upper(): - command += ' /p:Platform="ARM"' - - if targets: - command += " /target:%s" % ";".join(targets) - return command - - -def vs_installation_path(version): - if not hasattr(vs_installation_path, "_cached"): - vs_installation_path._cached = dict() - - if version not in vs_installation_path._cached: - vs_path = None - program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles")) - if program_files: - vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer", - "vswhere.exe") - if os.path.isfile(vswhere_path): - version_range = "[%d.0, %d.0)" % (int(version), int(version) + 1) - try: - output = subprocess.check_output([vswhere_path, "-version", version_range, - "-legacy", "-property", "installationPath"]) - vs_path = output.decode().strip() - _global_output.info("vswhere detected VS %s in %s" % (version, vs_path)) - except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e: - _global_output.error("vswhere error: %s" % str(e)) - - # Remember to cache result - vs_installation_path._cached[version] = vs_path - - return vs_installation_path._cached[version] - - -def vcvars_command(settings): - arch_setting = settings.get_safe("arch") - compiler_version = settings.get_safe("compiler.version") - if not compiler_version: - raise ConanException("compiler.version setting required for vcvars not defined") - - param = "x86" if arch_setting == "x86" else "amd64" - existing_version = os.environ.get("VisualStudioVersion") - if existing_version: - command = "echo Conan:vcvars already set" - existing_version = existing_version.split(".")[0] - if existing_version != compiler_version: - raise ConanException("Error, Visual environment already set to %s\n" - "Current settings visual version: %s" - % (existing_version, compiler_version)) - else: - env_var = "vs%s0comntools" % compiler_version - - if env_var == 'vs150comntools': - vs_path = os.getenv(env_var) - if not vs_path: # Try to locate with vswhere - vs_root = vs_installation_path("15") - if vs_root: - vs_path = os.path.join(vs_root, "Common7", "Tools") - else: - raise ConanException("VS2017 '%s' variable not defined, " - "and vswhere didn't find it" % env_var) - vcvars_path = os.path.join(vs_path, "../../VC/Auxiliary/Build/vcvarsall.bat") - command = ('set "VSCMD_START_DIR=%%CD%%" && ' - 'call "%s" %s' % (vcvars_path, param)) - else: - try: - vs_path = os.environ[env_var] - except KeyError: - raise ConanException("VS '%s' variable not defined. Please install VS" % env_var) - vcvars_path = os.path.join(vs_path, "../../VC/vcvarsall.bat") - command = ('call "%s" %s' % (vcvars_path, param)) - - return command - - -def cpu_count(): - try: - env_cpu_count = os.getenv("CONAN_CPU_COUNT", None) - return int(env_cpu_count) if env_cpu_count else multiprocessing.cpu_count() - except NotImplementedError: - _global_output.warn("multiprocessing.cpu_count() not implemented. Defaulting to 1 cpu") - return 1 # Safe guess - - -def human_size(size_bytes): - """ - format a size in bytes into a 'human' file size, e.g. B, KB, MB, GB, TB, PB - Note that bytes will be reported in whole numbers but KB and above will have - greater precision. e.g. 43 B, 443 KB, 4.3 MB, 4.43 GB, etc - """ - - suffixes_table = [('B', 0), ('KB', 1), ('MB', 1), ('GB', 2), ('TB', 2), ('PB', 2)] - - num = float(size_bytes) - for suffix, precision in suffixes_table: - if num < 1024.0: - break - num /= 1024.0 - - if precision == 0: - formatted_size = "%d" % num - else: - formatted_size = str(round(num, ndigits=precision)) - - return "%s%s" % (formatted_size, suffix) - - -def unzip(filename, destination=".", keep_permissions=False): - """ - Unzip a zipped file - :param filename: Path to the zip file - :param destination: Destination folder - :param keep_permissions: Keep the zip permissions. WARNING: Can be dangerous if the zip was not created in a NIX - system, the bits could produce undefined permission schema. Use only this option if you are sure that the - zip was created correctly. - :return: - """ - if (filename.endswith(".tar.gz") or filename.endswith(".tgz") or - filename.endswith(".tbz2") or filename.endswith(".tar.bz2") or - filename.endswith(".tar")): - return untargz(filename, destination) - import zipfile - full_path = os.path.normpath(os.path.join(os.getcwd(), destination)) - - if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): - def print_progress(extracted_size, uncompress_size): - txt_msg = "Unzipping %.0f %%" % (extracted_size * 100.0 / uncompress_size) - _global_output.rewrite_line(txt_msg) - else: - def print_progress(extracted_size, uncompress_size): - pass - - with zipfile.ZipFile(filename, "r") as z: - uncompress_size = sum((file_.file_size for file_ in z.infolist())) - if uncompress_size > 100000: - _global_output.info("Unzipping %s, this can take a while" % human_size(uncompress_size)) - else: - _global_output.info("Unzipping %s" % human_size(uncompress_size)) - extracted_size = 0 - if platform.system() == "Windows": - for file_ in z.infolist(): - extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) - try: - # Win path limit is 260 chars - if len(file_.filename) + len(full_path) >= 260: - raise ValueError("Filename too long") - z.extract(file_, full_path) - except Exception as e: - _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) - else: # duplicated for, to avoid a platform check for each zipped file - for file_ in z.infolist(): - extracted_size += file_.file_size - print_progress(extracted_size, uncompress_size) - try: - z.extract(file_, full_path) - if keep_permissions: - # Could be dangerous if the ZIP has been created in a non nix system - # https://bugs.python.org/issue15795 - perm = file_.external_attr >> 16 & 0xFFF - os.chmod(os.path.join(full_path, file_.filename), perm) - except Exception as e: - _global_output.error("Error extract %s\n%s" % (file_.filename, str(e))) - - -def untargz(filename, destination="."): - import tarfile - with tarfile.TarFile.open(filename, 'r:*') as tarredgzippedFile: - tarredgzippedFile.extractall(destination) - - -def get(url): - """ high level downloader + unziper + delete temporary zip - """ - filename = os.path.basename(url) - download(url, filename) - unzip(filename) - os.unlink(filename) - - -def ftp_download(ip, filename, login='', password=''): - import ftplib - try: - ftp = ftplib.FTP(ip, login, password) - ftp.login() - filepath, filename = os.path.split(filename) - if filepath: - ftp.cwd(filepath) - with open(filename, 'wb') as f: - ftp.retrbinary('RETR ' + filename, f.write) - except Exception as e: - raise ConanException("Error in FTP download from %s\n%s" % (ip, str(e))) - finally: - try: - ftp.quit() - except: - pass - - -def download(url, filename, verify=True, out=None, retry=2, retry_wait=5): - out = out or ConanOutput(sys.stdout, True) - if verify: - # We check the certificate using a list of known verifiers - import conans.client.rest.cacert as cacert - verify = cacert.file_path - downloader = Downloader(_global_requester, out, verify=verify) - downloader.download(url, filename, retry=retry, retry_wait=retry_wait) - out.writeln("") - - -# save(filename, content) - - -def replace_in_file(file_path, search, replace, strict=True): - content = load(file_path) - if -1 == content.find(search): - message = "replace_in_file didn't find pattern '%s' in '%s' file." % (search, file_path) - if strict: - raise ConanException(message) - else: - _global_output.warn(message) - content = content.replace(search, replace) - content = content.encode("utf-8") - with open(file_path, "wb") as handle: - handle.write(content) - - -def replace_prefix_in_pc_file(pc_file, new_prefix): - content = load(pc_file) - lines = [] - for line in content.splitlines(): - if line.startswith("prefix="): - lines.append('prefix=%s' % new_prefix) - else: - lines.append(line) - save(pc_file, "\n".join(lines)) - - -def check_with_algorithm_sum(algorithm_name, file_path, signature): - real_signature = _generic_algorithm_sum(file_path, algorithm_name) - if real_signature != signature: - raise ConanException("%s signature failed for '%s' file." - " Computed signature: %s" % (algorithm_name, - os.path.basename(file_path), - real_signature)) - - -def check_sha1(file_path, signature): - check_with_algorithm_sum("sha1", file_path, signature) - - -def check_md5(file_path, signature): - check_with_algorithm_sum("md5", file_path, signature) - - -def check_sha256(file_path, signature): - check_with_algorithm_sum("sha256", file_path, signature) - - -def patch(base_path=None, patch_file=None, patch_string=None, strip=0, output=None): - """Applies a diff from file (patch_file) or string (patch_string) - in base_path directory or current dir if None""" - - class PatchLogHandler(logging.Handler): - def __init__(self): - logging.Handler.__init__(self, logging.DEBUG) - self.output = output or ConanOutput(sys.stdout, True) - self.patchname = patch_file if patch_file else "patch" - - def emit(self, record): - logstr = self.format(record) - if record.levelno == logging.WARN: - self.output.warn("%s: %s" % (self.patchname, logstr)) - else: - self.output.info("%s: %s" % (self.patchname, logstr)) - - patchlog = logging.getLogger("patch") - if patchlog: - patchlog.handlers = [] - patchlog.addHandler(PatchLogHandler()) - - if not patch_file and not patch_string: - return - if patch_file: - patchset = fromfile(patch_file) - else: - patchset = fromstring(patch_string.encode()) - - if not patchset: - raise ConanException("Failed to parse patch: %s" % (patch_file if patch_file else "string")) - - if not patchset.apply(root=base_path, strip=strip): - raise ConanException("Failed to apply patch: %s" % patch_file) - - -def cross_building(settings, self_os=None, self_arch=None): - self_os = self_os or platform.system() - self_arch = self_arch or detected_architecture() - os_setting = settings.get_safe("os") - arch_setting = settings.get_safe("arch") - platform_os = {"Darwin": "Macos"}.get(self_os, self_os) - if self_os == os_setting and self_arch == "x86_64" and arch_setting == "x86": - return False # not really considered cross - - if os_setting and platform_os != os_setting: - return True - if arch_setting and self_arch != arch_setting: - return True - - return False - - -def detected_architecture(): - # FIXME: Very weak check but not very common to run conan in other architectures - if "64" in platform.machine(): - return "x86_64" - elif "86" in platform.machine(): - return "x86" - return None - - -# DETECT OS, VERSION AND DISTRIBUTIONS - -class OSInfo(object): - """ Usage: - (os_info.is_linux) # True/False - (os_info.is_windows) # True/False - (os_info.is_macos) # True/False - (os_info.is_freebsd) # True/False - (os_info.is_solaris) # True/False - - (os_info.linux_distro) # debian, ubuntu, fedora, centos... - - (os_info.os_version) # 5.1 - (os_info.os_version_name) # Windows 7, El Capitan - - if os_info.os_version > "10.1": - pass - if os_info.os_version == "10.1.0": - pass - """ - - def __init__(self): - self.os_version = None - self.os_version_name = None - self.is_linux = platform.system() == "Linux" - self.linux_distro = None - self.is_windows = platform.system() == "Windows" - self.is_macos = platform.system() == "Darwin" - self.is_freebsd = platform.system() == "FreeBSD" - self.is_solaris = platform.system() == "SunOS" - - if self.is_linux: - import distro - self.linux_distro = distro.id() - self.os_version = Version(distro.version()) - version_name = distro.codename() - self.os_version_name = version_name if version_name != "n/a" else "" - if not self.os_version_name and self.linux_distro == "debian": - self.os_version_name = self.get_debian_version_name(self.os_version) - elif self.is_windows: - self.os_version = self.get_win_os_version() - self.os_version_name = self.get_win_version_name(self.os_version) - elif self.is_macos: - self.os_version = Version(platform.mac_ver()[0]) - self.os_version_name = self.get_osx_version_name(self.os_version) - elif self.is_freebsd: - self.os_version = self.get_freebsd_version() - self.os_version_name = "FreeBSD %s" % self.os_version - elif self.is_solaris: - self.os_version = Version(platform.release()) - self.os_version_name = self.get_solaris_version_name(self.os_version) - - @property - def with_apt(self): - return self.is_linux and self.linux_distro in \ - ("debian", "ubuntu", "knoppix", "linuxmint", "raspbian") - - @property - def with_yum(self): - return self.is_linux and self.linux_distro in \ - ("centos", "redhat", "fedora", "pidora", "scientific", - "xenserver", "amazon", "oracle", "rhel") - - @staticmethod - def get_win_os_version(): - """ - Get's the OS major and minor versions. Returns a tuple of - (OS_MAJOR, OS_MINOR). - """ - import ctypes - - class _OSVERSIONINFOEXW(ctypes.Structure): - _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), - ('dwMajorVersion', ctypes.c_ulong), - ('dwMinorVersion', ctypes.c_ulong), - ('dwBuildNumber', ctypes.c_ulong), - ('dwPlatformId', ctypes.c_ulong), - ('szCSDVersion', ctypes.c_wchar * 128), - ('wServicePackMajor', ctypes.c_ushort), - ('wServicePackMinor', ctypes.c_ushort), - ('wSuiteMask', ctypes.c_ushort), - ('wProductType', ctypes.c_byte), - ('wReserved', ctypes.c_byte)] - - os_version = _OSVERSIONINFOEXW() - os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) - retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) - if retcode != 0: - return None - - return Version("%d.%d" % (os_version.dwMajorVersion, os_version.dwMinorVersion)) - - @staticmethod - def get_debian_version_name(version): - if not version: - return None - elif version.major() == "8.Y.Z": - return "jessie" - elif version.major() == "7.Y.Z": - return "wheezy" - elif version.major() == "6.Y.Z": - return "squeeze" - elif version.major() == "5.Y.Z": - return "lenny" - elif version.major() == "4.Y.Z": - return "etch" - elif version.minor() == "3.1.Z": - return "sarge" - elif version.minor() == "3.0.Z": - return "woody" - - @staticmethod - def get_win_version_name(version): - if not version: - return None - elif version.major() == "5.Y.Z": - return "Windows XP" - elif version.minor() == "6.0.Z": - return "Windows Vista" - elif version.minor() == "6.1.Z": - return "Windows 7" - elif version.minor() == "6.2.Z": - return "Windows 8" - elif version.minor() == "6.3.Z": - return "Windows 8.1" - elif version.minor() == "10.0.Z": - return "Windows 10" - - @staticmethod - def get_osx_version_name(version): - if not version: - return None - elif version.minor() == "10.12.Z": - return "Sierra" - elif version.minor() == "10.11.Z": - return "El Capitan" - elif version.minor() == "10.10.Z": - return "Yosemite" - elif version.minor() == "10.9.Z": - return "Mavericks" - elif version.minor() == "10.8.Z": - return "Mountain Lion" - elif version.minor() == "10.7.Z": - return "Lion" - elif version.minor() == "10.6.Z": - return "Snow Leopard" - elif version.minor() == "10.5.Z": - return "Leopard" - elif version.minor() == "10.4.Z": - return "Tiger" - elif version.minor() == "10.3.Z": - return "Panther" - elif version.minor() == "10.2.Z": - return "Jaguar" - elif version.minor() == "10.1.Z": - return "Puma" - elif version.minor() == "10.0.Z": - return "Cheetha" - - @staticmethod - def get_freebsd_version(): - return platform.release().split("-")[0] - - @staticmethod - def get_solaris_version_name(version): - if not version: - return None - elif version.minor() == "5.10": - return "Solaris 10" - elif version.minor() == "5.11": - return "Solaris 11" - - -try: - os_info = OSInfo() -except Exception as exc: - logger.error(exc) - _global_output.error("Error detecting os_info") - - -class SystemPackageTool(object): - - def __init__(self, runner=None, os_info=None, tool=None): - env_sudo = os.environ.get("CONAN_SYSREQUIRES_SUDO", None) - self._sudo = (env_sudo != "False" and env_sudo != "0") - os_info = os_info or OSInfo() - self._is_up_to_date = False - self._tool = tool or self._create_tool(os_info) - self._tool._sudo_str = "sudo " if self._sudo else "" - self._tool._runner = runner or ConanRunner() - - def _create_tool(self, os_info): - if os_info.with_apt: - return AptTool() - elif os_info.with_yum: - return YumTool() - elif os_info.is_macos: - return BrewTool() - elif os_info.is_freebsd: - return PkgTool() - elif os_info.is_solaris: - return PkgUtilTool() - else: - return NullTool() - - def update(self): - """ - Get the system package tool update command - """ - self._is_up_to_date = True - self._tool.update() - - def install(self, packages, update=True, force=False): - ''' - Get the system package tool install command. - ''' - packages = [packages] if isinstance(packages, str) else list(packages) - if not force and self._installed(packages): - return - if update and not self._is_up_to_date: - self.update() - self._install_any(packages) - - def _installed(self, packages): - for pkg in packages: - if self._tool.installed(pkg): - _global_output.info("Package already installed: %s" % pkg) - return True - return False - - def _install_any(self, packages): - if len(packages) == 1: - return self._tool.install(packages[0]) - for pkg in packages: - try: - return self._tool.install(pkg) - except ConanException: - pass - raise ConanException("Could not install any of %s" % packages) - - -class NullTool(object): - def update(self): - pass - - def install(self, package_name): - _global_output.warn("Only available for linux with apt-get or yum or OSx with brew or FreeBSD with pkg or Solaris with pkgutil") - - def installed(self, package_name): - return False - - -class AptTool(object): - def update(self): - _run(self._runner, "%sapt-get update" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%sapt-get install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("dpkg -s %s" % package_name, None) - return exit_code == 0 - - -class YumTool(object): - def update(self): - _run(self._runner, "%syum check-update" % self._sudo_str, accepted_returns=[0, 100]) - - def install(self, package_name): - _run(self._runner, "%syum install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("rpm -q %s" % package_name, None) - return exit_code == 0 - - -class BrewTool(object): - def update(self): - _run(self._runner, "brew update") - - def install(self, package_name): - _run(self._runner, "brew install %s" % package_name) - - def installed(self, package_name): - exit_code = self._runner('test -n "$(brew ls --versions %s)"' % package_name, None) - return exit_code == 0 - - -class PkgTool(object): - def update(self): - _run(self._runner, "%spkg update" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%spkg install -y %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner("pkg info %s" % package_name, None) - return exit_code == 0 - - -class PkgUtilTool(object): - def update(self): - _run(self._runner, "%spkgutil --catalog" % self._sudo_str) - - def install(self, package_name): - _run(self._runner, "%spkgutil --install --yes %s" % (self._sudo_str, package_name)) - - def installed(self, package_name): - exit_code = self._runner('test -n "`pkgutil --list %s`"' % package_name, None) - return exit_code == 0 +from conans.util.files import (_generic_algorithm_sum, load, save, sha256sum, + sha1sum, md5sum, md5, touch, relative_dirs, rmdir, mkdir) -class ChocolateyTool(object): - def update(self): - _run(self._runner, "choco outdated") - def install(self, package_name): - _run(self._runner, "choco install --yes %s" % package_name) +# Global instances +def set_global_instances(the_output, the_requester): + # Assign global variables to needed modules + from conans.client.tools import files as _files + from conans.client.tools import net as _net + from conans.client.tools import oss as _oss + from conans.client.tools import system_pm as _system_pm + from conans.client.tools import win as _win - def installed(self, package_name): - exit_code = self._runner('choco search --local-only --exact %s | findstr /c:"1 packages installed."' % package_name, None) - return exit_code == 0 + _files._global_output = the_output + _oss._global_output = the_output + _system_pm._global_output = the_output + _win._global_output = the_output + _net._global_requester = the_requester -def _run(runner, command, accepted_returns=None): - accepted_returns = accepted_returns or [0, ] - _global_output.info("Running: %s" % command) - if runner(command, True) not in accepted_returns: - raise ConanException("Command '%s' failed" % command) +set_global_instances(ConanOutput(sys.stdout), requests) From 6faf40719504669309617d605f92f350c6ae578b Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Mon, 18 Sep 2017 17:04:14 +0200 Subject: [PATCH 31/39] Try CI (#1763) --- conans/client/command.py | 7 +++++++ conans/client/conan_api.py | 20 ++++++++++++++++++++ conans/client/conan_command_output.py | 3 +++ conans/test/command/profile_test.py | 20 +++++++++++++++++++- 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/conans/client/command.py b/conans/client/command.py index f22dd222f9c..b6a4aca99e2 100644 --- a/conans/client/command.py +++ b/conans/client/command.py @@ -765,6 +765,10 @@ def profile(self, *args): parser_update.add_argument('item', help='key="value to set", e.j: settings.compiler=gcc') parser_update.add_argument('profile', help='name of the profile') + parser_get = subparsers.add_parser('get', help='Get a profile key') + parser_get.add_argument('item', help='key="value to get", e.j: settings.compiler') + parser_get.add_argument('profile', help='name of the profile') + parser_remove = subparsers.add_parser('remove', help='Remove a profile key') parser_remove.add_argument('item', help='key", e.j: settings.compiler') parser_remove.add_argument('profile', help='name of the profile') @@ -787,6 +791,9 @@ def profile(self, *args): except: raise ConanException("Please specify key=value") self._conan.update_profile(profile, key, value) + elif args.subcommand == "get": + key = args.item + self._outputer.writeln(self._conan.get_profile_key(profile, key)) elif args.subcommand == "remove": self._conan.delete_profile_key(profile, args.item) diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index a420220e598..3077fe6d281 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -690,6 +690,26 @@ def update_profile(self, profile_name, key, value): profile_path = get_profile_path(profile_name, self._client_cache.profiles_path, os.getcwd()) save(profile_path, contents) + @api_method + def get_profile_key(self, profile_name, key): + first_key, rest_key = self._get_profile_keys(key) + profile, _ = read_profile(profile_name, os.getcwd(), self._client_cache.profiles_path) + try: + if first_key == "settings": + return profile.settings[rest_key] + elif first_key == "options": + return dict(profile.options.as_list())[rest_key] + elif first_key == "env": + package = None + var = rest_key + if ":" in rest_key: + package, var = rest_key.split(":") + return profile.env_values.data[package][var] + elif first_key == "build_requires": + raise ConanException("List the profile manually to see the build_requires") + except KeyError: + raise ConanException("Key not found: '%s'" % key) + @api_method def delete_profile_key(self, profile_name, key): first_key, rest_key = self._get_profile_keys(key) diff --git a/conans/client/conan_command_output.py b/conans/client/conan_command_output.py index 169d030a3cd..a8100212935 100644 --- a/conans/client/conan_command_output.py +++ b/conans/client/conan_command_output.py @@ -14,6 +14,9 @@ def __init__(self, user_io, client_cache): self.user_io = user_io self.client_cache = client_cache + def writeln(self, value): + self.user_io.out.writeln(value) + def print_profile(self, profile, profile_text): Printer(self.user_io.out).print_profile(profile, profile_text) diff --git a/conans/test/command/profile_test.py b/conans/test/command/profile_test.py index 9429c8c3fd0..cdcf9b1dcc2 100644 --- a/conans/test/command/profile_test.py +++ b/conans/test/command/profile_test.py @@ -41,7 +41,7 @@ def show_test(self): self.assertIn(" CXX=/path/tomy/g++_build", client.user_io.out) self.assertIn(" package:VAR=value", client.user_io.out) - def profile_update_test(self): + def profile_update_and_get_test(self): client = TestClient() client.run("profile new ./MyProfile --detect") pr_path = os.path.join(client.current_folder, "MyProfile") @@ -50,18 +50,33 @@ def profile_update_test(self): self.assertIn("os=FakeOS", load(pr_path)) self.assertNotIn("os=Linux", load(pr_path)) + client.run("profile get settings.os ./MyProfile") + self.assertEquals(client.out, "FakeOS\n") + client.run("profile update settings.compiler.version=88 ./MyProfile") self.assertIn("compiler.version=88", load(pr_path)) + client.run("profile get settings.compiler.version ./MyProfile") + self.assertEquals(client.out, "88\n") + client.run("profile update options.MyOption=23 ./MyProfile") self.assertIn("[options]\nMyOption=23", load(pr_path)) + client.run("profile get options.MyOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update options.Package:MyOption=23 ./MyProfile") self.assertIn("Package:MyOption=23", load(pr_path)) + client.run("profile get options.Package:MyOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update options.Package:OtherOption=23 ./MyProfile") self.assertIn("Package:OtherOption=23", load(pr_path)) + client.run("profile get options.Package:OtherOption ./MyProfile") + self.assertEquals(client.out, "23\n") + client.run("profile update scopes.Package:OneScope=True ./MyProfile") self.assertIn("[scopes]\nPackage:OneScope=True", load(pr_path)) @@ -72,6 +87,9 @@ def profile_update_test(self): client.run("profile update env.OneMyEnv=MYVALUe ./MyProfile") self.assertIn("[env]\nOneMyEnv=MYVALUe", load(pr_path)) + client.run("profile get env.OneMyEnv ./MyProfile") + self.assertEquals(client.out, "MYVALUe\n") + # Now try the remove client.run("profile remove settings.os ./MyProfile") From c28432495edacf888a04ec2ff7b6aec4f776bf24 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 19 Sep 2017 09:50:12 +0200 Subject: [PATCH 32/39] Fixed unzip with files without contents (#1765) --- conans/client/tools/files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py index a52fe94486c..9601b899f0b 100644 --- a/conans/client/tools/files.py +++ b/conans/client/tools/files.py @@ -69,7 +69,8 @@ def unzip(filename, destination=".", keep_permissions=False): if hasattr(sys.stdout, "isatty") and sys.stdout.isatty(): def print_progress(the_size, uncomp_size): - txt_msg = "Unzipping %.0f %%" % (the_size * 100.0 / uncomp_size) + the_size = (the_size * 100.0 / uncomp_size) if uncomp_size != 0 else 0 + txt_msg = "Unzipping %.0f %%" % the_size _global_output.rewrite_line(txt_msg) else: def print_progress(_, __): From 06fcb03743e7789ff23c5d987b3e594cce66dc25 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 19 Sep 2017 11:34:45 +0200 Subject: [PATCH 33/39] added NO_PROXY for python-requests (#1370) * added NO_PROXY for python-requests * disable proxies with None value * fixed test * added test * proxies portable py3 --- conans/client/conan_api.py | 8 +++++- conans/client/conf/__init__.py | 8 +++--- conans/test/functional/proxies_conf_test.py | 29 +++++++++++++++++++++ conans/test/util/client_conf_test.py | 13 +++++++++ 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 conans/test/functional/proxies_conf_test.py diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py index 3077fe6d281..819729aaea4 100644 --- a/conans/client/conan_api.py +++ b/conans/client/conan_api.py @@ -43,7 +43,13 @@ def get_basic_requester(client_cache): requester = requests.Session() - requester.proxies = client_cache.conan_config.proxies + proxies = client_cache.conan_config.proxies + if proxies: + # Account for the requests NO_PROXY env variable, not defined as a proxy like http= + no_proxy = proxies.pop("no_proxy", None) + if no_proxy: + os.environ["NO_PROXY"] = no_proxy + requester.proxies = proxies return requester diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 6bbf43a4f74..c52a9ed0062 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -1,7 +1,7 @@ import os -import urllib from six.moves.configparser import ConfigParser, NoSectionError +from six.moves import urllib from conans.errors import ConanException from conans.model.env_info import unquote @@ -150,6 +150,7 @@ def env_vars(self): "CONAN_BASH_PATH": self._env_c("general.bash_path", "CONAN_BASH_PATH", None), } + # Filter None values return {name: value for name, value in ret.items() if value is not None} @@ -273,7 +274,8 @@ def proxies(self): proxies = self.get_conf("proxies") # If there is proxies section, but empty, it will try to use system proxy if not proxies: - return urllib.getproxies() - return dict(proxies) + return urllib.request.getproxies() + result = {k: (None if v == "None" else v) for k, v in proxies} + return result except: return None diff --git a/conans/test/functional/proxies_conf_test.py b/conans/test/functional/proxies_conf_test.py new file mode 100644 index 00000000000..8f2beaa7850 --- /dev/null +++ b/conans/test/functional/proxies_conf_test.py @@ -0,0 +1,29 @@ +import unittest +import os + +from conans.test.utils.tools import TestClient +from conans.util.files import save +from conans.client.conan_api import get_basic_requester + + +class ProxiesConfTest(unittest.TestCase): + def setUp(self): + self.old_env = dict(os.environ) + + def tearDown(self): + os.environ.clear() + os.environ.update(self.old_env) + + def test_requester(self): + client = TestClient() + conf = """ +[proxies] +https=None +no_proxy=http://someurl,http://otherurl.com +http=http:/conan.url + """ + save(client.client_cache.conan_conf_path, conf) + requester = get_basic_requester(client.client_cache) + self.assertEqual(requester.proxies, {"https": None, + "http": "http:/conan.url"}) + self.assertEqual(os.environ["NO_PROXY"], "http://someurl,http://otherurl.com") diff --git a/conans/test/util/client_conf_test.py b/conans/test/util/client_conf_test.py index 74e57df1a49..78588189ebb 100644 --- a/conans/test/util/client_conf_test.py +++ b/conans/test/util/client_conf_test.py @@ -7,6 +7,7 @@ from conans.test.utils.test_files import temp_folder from conans.util.files import save + default_client_conf = '''[storage] path: ~/.conan/data @@ -37,3 +38,15 @@ def test_quotes(self): save(os.path.join(tmp_dir, DEFAULT_PROFILE_NAME), default_profile) config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) self.assertEqual(config.env_vars["CONAN_TRACE_FILE"], "Path/with/quotes") + + def test_proxies(self): + tmp_dir = temp_folder() + save(os.path.join(tmp_dir, CONAN_CONF), "") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertEqual(None, config.proxies) + save(os.path.join(tmp_dir, CONAN_CONF), "[proxies]") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertNotIn("no_proxy", config.proxies) + save(os.path.join(tmp_dir, CONAN_CONF), "[proxies]\nno_proxy=localhost") + config = ConanClientConfigParser(os.path.join(tmp_dir, CONAN_CONF)) + self.assertEqual(config.proxies["no_proxy"], "localhost") From 9c70e798689bd6c66dbce82428226993945c7af2 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 19 Sep 2017 16:40:08 +0200 Subject: [PATCH 34/39] Feature/manifest path (#1772) * Changed the log message on the ManifestManager to add the path of the manifest files to the log. Added name to contributors txt. * fixed broken tests --- conans/client/manifest_manager.py | 2 +- conans/test/functional/proxies_conf_test.py | 2 +- conans/test/integration/manifest_validation_test.py | 2 +- contributors.txt | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conans/client/manifest_manager.py b/conans/client/manifest_manager.py index ce566375a25..40899b74f14 100644 --- a/conans/client/manifest_manager.py +++ b/conans/client/manifest_manager.py @@ -18,7 +18,7 @@ def __init__(self, folder, user_io, client_cache, verify=False, interactive=Fals self._log = [] def print_log(self): - self._user_io.out.success("\nManifests") + self._user_io.out.success("\nManifests : %s" % (self._paths.store)) for log_entry in self._log: self._user_io.out.info(log_entry) diff --git a/conans/test/functional/proxies_conf_test.py b/conans/test/functional/proxies_conf_test.py index 8f2beaa7850..8e0b3e89d43 100644 --- a/conans/test/functional/proxies_conf_test.py +++ b/conans/test/functional/proxies_conf_test.py @@ -15,7 +15,7 @@ def tearDown(self): os.environ.update(self.old_env) def test_requester(self): - client = TestClient() + client = TestClient(default_profile=False) conf = """ [proxies] https=None diff --git a/conans/test/integration/manifest_validation_test.py b/conans/test/integration/manifest_validation_test.py index 177535f330e..ddf69f6ef39 100644 --- a/conans/test/integration/manifest_validation_test.py +++ b/conans/test/integration/manifest_validation_test.py @@ -92,7 +92,7 @@ def _capture_verify_manifest(self, reference, remote="local cache", folder=""): # again should do nothing self.client.run("install %s --build missing --manifests %s" % (str(self.reference), folder)) - self.assertNotIn("manifest", self.client.user_io.out) + self.assertNotIn("Installed manifest", self.client.user_io.out) # now verify self.client.run("install %s --build missing --verify %s" % (str(self.reference), folder)) diff --git a/contributors.txt b/contributors.txt index 5f039fc93b1..857e589da6b 100644 --- a/contributors.txt +++ b/contributors.txt @@ -16,6 +16,7 @@ Many thanks to all of them! - Koch, Marco (marco-koch@t-online.de, @MarcoKoch) - Kourkoulis, Dimitri (@dimi309) - Lee, Jeongseok (jslee02@gmail.com, @jslee02) +- Lord, Matthew( @luckielordie ) - Márki, Róbert (gsmiko@gmail.com, @robertmrk) - Ray, Chris (chris@xaltotun.com) - Ries, Uilian (uilianries@gmail.com, @uilianries) From 45fd9197aa88f2d92a19f8538b6ebe4ef897d44a Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 19 Sep 2017 16:42:09 +0200 Subject: [PATCH 35/39] Do not upload tgzs with .c_src (#1762) * Do not upload tgzs with .c_src * removed var * Limit min compatible version to 0.25.0 --- conans/client/migrations.py | 3 ++- conans/client/remote_manager.py | 9 +++------ conans/paths.py | 3 +-- conans/server/conf/__init__.py | 2 +- ...est.py => remove_old_export_sources_layout_test.py} | 10 +++++----- 5 files changed, 12 insertions(+), 15 deletions(-) rename conans/test/{keep_old_export_sources_layout_test.py => remove_old_export_sources_layout_test.py} (84%) diff --git a/conans/client/migrations.py b/conans/client/migrations.py index bec3d432c57..cbb23278a20 100644 --- a/conans/client/migrations.py +++ b/conans/client/migrations.py @@ -4,6 +4,7 @@ from conans.client.client_cache import CONAN_CONF, PROFILES_FOLDER from conans.errors import ConanException from conans.migrations import Migrator, CONAN_VERSION +from conans.paths import EXPORT_SOURCES_DIR_OLD from conans.util.files import load, save from conans.model.version import Version @@ -131,7 +132,7 @@ def migrate_c_src_export_source(client_cache, out): package_folders = list_folder_subdirs(client_cache.store, 4) for package in package_folders: package_folder = os.path.join(client_cache.store, package) - c_src = os.path.join(package_folder, "export/.c_src") + c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD) if os.path.exists(c_src): out.warn("Migration: Removing package with old export_sources layout: %s" % package) try: diff --git a/conans/client/remote_manager.py b/conans/client/remote_manager.py index b1155026384..4927d1b51be 100644 --- a/conans/client/remote_manager.py +++ b/conans/client/remote_manager.py @@ -8,8 +8,8 @@ from conans.errors import ConanException, ConanConnectionError, NotFoundException from conans.model.manifest import gather_files -from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME,\ - rm_conandir, EXPORT_SOURCES_TGZ_NAME +from conans.paths import PACKAGE_TGZ_NAME, CONANINFO, CONAN_MANIFEST, CONANFILE, EXPORT_TGZ_NAME, \ + rm_conandir, EXPORT_SOURCES_TGZ_NAME, EXPORT_SOURCES_DIR_OLD from conans.util.files import gzopen_without_timestamps from conans.util.files import tar_extract, rmdir, exception_message_safe, mkdir from conans.util.files import touch @@ -193,7 +193,7 @@ def filter_function(urls): return unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME) - c_src_path = os.path.join(export_sources_folder, ".c_src") + c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD) if os.path.exists(c_src_path): merge_directories(c_src_path, export_sources_folder) rmdir(c_src_path) @@ -301,7 +301,6 @@ def compress_files(files, symlinks, name, dest_dir): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) - is_export_sources = (name == EXPORT_SOURCES_TGZ_NAME) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) @@ -313,8 +312,6 @@ def compress_files(files, symlinks, name, dest_dir): tgz.addfile(tarinfo=info) for filename, abs_path in files.items(): - if is_export_sources: # temporary backwards compat TGZ creation - filename = ".c_src/%s" % filename info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode diff --git a/conans/paths.py b/conans/paths.py index 907c928273c..eb028b3c8fc 100644 --- a/conans/paths.py +++ b/conans/paths.py @@ -41,8 +41,7 @@ def path_shortener(x, _): PACKAGE_TGZ_NAME = "conan_package.tgz" EXPORT_TGZ_NAME = "conan_export.tgz" EXPORT_SOURCES_TGZ_NAME = "conan_sources.tgz" -EXPORT_SOURCES_DIR = ".c_src" - +EXPORT_SOURCES_DIR_OLD = ".c_src" RUN_LOG_NAME = "conan_run.log" diff --git a/conans/server/conf/__init__.py b/conans/server/conf/__init__.py index 48e06b3f5de..5d6fff511a6 100644 --- a/conans/server/conf/__init__.py +++ b/conans/server/conf/__init__.py @@ -16,7 +16,7 @@ from conans.util.log import logger from conans.server.conf.default_server_conf import default_server_conf -MIN_CLIENT_COMPATIBLE_VERSION = '0.19.3' +MIN_CLIENT_COMPATIBLE_VERSION = '0.25.0' class ConanServerConfigParser(ConfigParser): diff --git a/conans/test/keep_old_export_sources_layout_test.py b/conans/test/remove_old_export_sources_layout_test.py similarity index 84% rename from conans/test/keep_old_export_sources_layout_test.py rename to conans/test/remove_old_export_sources_layout_test.py index bb48f40038b..e2e082f8cc3 100644 --- a/conans/test/keep_old_export_sources_layout_test.py +++ b/conans/test/remove_old_export_sources_layout_test.py @@ -1,16 +1,17 @@ import unittest import os +from conans.paths import EXPORT_SOURCES_DIR_OLD from conans.util.files import tar_extract from conans.test.utils.tools import TestServer, TestClient from conans.model.ref import ConanFileReference from conans.test.utils.test_files import temp_folder -class KeepOldExportSourcesLayoutTest(unittest.TestCase): +class DoNotKeepOldExportSourcesLayoutTest(unittest.TestCase): def test_basic(self): - """ check that we generate tgz with .c_src and we handle them properly + """ check that we do not generate anymore tgz with .c_src. also, they are not present any more in the cache layout, even if they come from a .c_src tgz server file """ @@ -35,11 +36,10 @@ class MyPkg(ConanFile): folder = temp_folder() with open(sources_tgz, 'rb') as file_handler: tar_extract(file_handler, folder) - self.assertEqual(os.listdir(folder), [".c_src"]) + self.assertEqual(os.listdir(folder), ["myfile.txt"]) # Now install again client.run("install Pkg/0.1@lasote/testing --build=missing") export = client.client_cache.export(conan_reference) - self.assertNotIn(".c_src", os.listdir(export)) + self.assertNotIn(EXPORT_SOURCES_DIR_OLD, os.listdir(export)) export_sources = client.client_cache.export_sources(conan_reference) self.assertEqual(os.listdir(export_sources), ["myfile.txt"]) - \ No newline at end of file From ed28016555f94867cb3c142be9daaf2e551019c6 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 19 Sep 2017 16:42:42 +0200 Subject: [PATCH 36/39] Feature/new requires (#1751) * removed dev_requires * simplified * removed traceback print --- conans/client/deps_builder.py | 27 +---------- conans/model/conan_file.py | 10 ---- conans/model/info.py | 30 ++++-------- conans/model/options.py | 13 ++---- conans/model/requires.py | 45 ++++-------------- conans/test/command/info_test.py | 19 ++------ conans/test/integration/conan_scopes_test.py | 48 +------------------- conans/test/model/options_test.py | 4 +- 8 files changed, 30 insertions(+), 166 deletions(-) diff --git a/conans/client/deps_builder.py b/conans/client/deps_builder.py index 6bad0b09d07..3c0143ed1da 100644 --- a/conans/client/deps_builder.py +++ b/conans/client/deps_builder.py @@ -118,13 +118,10 @@ def propagate_info(self): conanfile.build_requires_options = conanfile.options.values conanfile.options.clear_unused(indirect_reqs.union(direct_reqs)) - non_devs = self.non_dev_nodes(node) - conanfile.info = ConanInfo.create(conanfile.settings.values, conanfile.options.values, direct_reqs, - indirect_reqs, - non_devs) + indirect_reqs) # Once we are done, call package_id() to narrow and change possible values if hasattr(conanfile, "conan_info"): @@ -249,28 +246,6 @@ def private_nodes(self, built_private_nodes): result.append(node) return result - def non_dev_nodes(self, root): - if not root.conanfile.scope.dev: - # Optimization. This allow not to check it for most packages, which dev=False - return None - open_nodes = set([root]) - result = set() - expanded = set() - while open_nodes: - new_open_nodes = set() - for node in open_nodes: - neighbors = self._neighbors[node] - requires = node.conanfile.requires - for n in neighbors: - requirement = requires[n.conan_ref.name] - if not requirement.dev and n not in expanded: - result.add(n.conan_ref.name) - new_open_nodes.add(n) - expanded.add(n) - - open_nodes = new_open_nodes - return result - class DepsGraphBuilder(object): """ Responsible for computing the dependencies graph DepsGraph diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 57cffa8e2b1..2a4db5fcdd7 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -169,16 +169,6 @@ def scope(self): @scope.setter def scope(self, value): self._scope = value - if value.dev: - self.requires.allow_dev = True - try: - if hasattr(self, "dev_requires"): - if isinstance(self.dev_requires, tuple): - self.requires.add_dev(*self.dev_requires) - else: - self.requires.add_dev(self.dev_requires, ) - except Exception as e: - raise ConanException("Error while initializing dev_requirements. %s" % str(e)) @property def conanfile_directory(self): diff --git a/conans/model/info.py b/conans/model/info.py index 7cfca9e29d0..fecae68326d 100644 --- a/conans/model/info.py +++ b/conans/model/info.py @@ -101,14 +101,12 @@ def full_package_mode(self): class RequirementsInfo(object): - def __init__(self, requires, non_devs_requirements): + def __init__(self, requires): # {PackageReference: RequirementInfo} - self._non_devs_requirements = non_devs_requirements self._data = {r: RequirementInfo(str(r)) for r in requires} def copy(self): - return RequirementsInfo(self._data.keys(), self._non_devs_requirements.copy() - if self._non_devs_requirements else None) + return RequirementsInfo(self._data.keys()) def clear(self): self._data = {} @@ -151,14 +149,8 @@ def sha(self): result = [] # Remove requirements without a name, i.e. indirect transitive requirements data = {k: v for k, v in self._data.items() if v.name} - if self._non_devs_requirements is None: - for key in sorted(data): - result.append(data[key].sha) - else: - for key in sorted(data): - non_dev = key.conan.name in self._non_devs_requirements - if non_dev: - result.append(data[key].sha) + for key in sorted(data): + result.append(data[key].sha) return sha1('\n'.join(result).encode()) def dumps(self): @@ -166,10 +158,6 @@ def dumps(self): for ref in sorted(self._data): dumped = self._data[ref].dumps() if dumped: - dev = (self._non_devs_requirements is not None and - ref.conan.name not in self._non_devs_requirements) - if dev: - dumped += " DEV" result.append(dumped) return "\n".join(result) @@ -245,11 +233,10 @@ def copy(self): result.settings = self.settings.copy() result.options = self.options.copy() result.requires = self.requires.copy() - result._non_devs_requirements = self._non_devs_requirements return result @staticmethod - def create(settings, options, requires, indirect_requires, non_devs_requirements): + def create(settings, options, requires, indirect_requires): result = ConanInfo() result.full_settings = settings result.settings = settings.copy() @@ -257,12 +244,11 @@ def create(settings, options, requires, indirect_requires, non_devs_requirements result.options = options.copy() result.options.clear_indirect() result.full_requires = RequirementsList(requires) - result.requires = RequirementsInfo(requires, non_devs_requirements) + result.requires = RequirementsInfo(requires) result.scope = None result.requires.add(indirect_requires) result.full_requires.extend(indirect_requires) result.recipe_hash = None - result._non_devs_requirements = non_devs_requirements # Can be None result.env_values = EnvValues() return result @@ -277,7 +263,7 @@ def loads(text): result.options = OptionsValues.loads(parser.options) result.full_options = OptionsValues.loads(parser.full_options) result.full_requires = RequirementsList.loads(parser.full_requires) - result.requires = RequirementsInfo(result.full_requires, None) + result.requires = RequirementsInfo(result.full_requires) result.recipe_hash = parser.recipe_hash or None # TODO: Missing handling paring of requires, but not necessary now @@ -344,7 +330,7 @@ def package_id(self): # Only are valid requires for OPtions those Non-Dev who are still in requires self.options.filter_used(self.requires.pkg_names) - result.append(self.options.sha(self._non_devs_requirements)) + result.append(self.options.sha) result.append(self.requires.sha) self._package_id = sha1('\n'.join(result).encode()) return self._package_id diff --git a/conans/model/options.py b/conans/model/options.py index 54d43672fe0..b3b7843f20e 100644 --- a/conans/model/options.py +++ b/conans/model/options.py @@ -254,17 +254,12 @@ def loads(text): result.append((name.strip(), value.strip())) return OptionsValues(result) - def sha(self, non_dev_requirements): + @property + def sha(self): result = [] result.append(self._package_values.sha) - if non_dev_requirements is None: # Not filtering - for key in sorted(list(self._reqs_options.keys())): - result.append(self._reqs_options[key].sha) - else: - for key in sorted(list(self._reqs_options.keys())): - non_dev = key in non_dev_requirements - if non_dev: - result.append(self._reqs_options[key].sha) + for key in sorted(list(self._reqs_options.keys())): + result.append(self._reqs_options[key].sha) return sha1('\n'.join(result).encode()) def serialize(self): diff --git a/conans/model/requires.py b/conans/model/requires.py index 5c7a221262e..ad7e1ae6ba5 100644 --- a/conans/model/requires.py +++ b/conans/model/requires.py @@ -8,20 +8,15 @@ class Requirement(object): """ A reference to a package plus some attributes of how to depend on that package """ - def __init__(self, conan_reference, private=False, override=False, dev=False): + def __init__(self, conan_reference, private=False, override=False): """ param override: True means that this is not an actual requirement, but something to be passed upstream and override possible existing values - param private: True means that this requirement will be somewhat embedded (like - a static lib linked into a shared lib), so it is not required to link - param dev: True means that this requirement is only needed at dev time, e.g. only - needed for building or testing, but not affects the package hash at all """ self.conan_reference = conan_reference self.range_reference = conan_reference - self.private = private self.override = override - self.dev = dev + self.private = private @property def version_range(self): @@ -45,8 +40,7 @@ def __repr__(self): def __eq__(self, other): return (self.override == other.override and self.conan_reference == other.conan_reference and - self.private == other.private and - self.dev == other.dev) + self.private == other.private) def __ne__(self, other): return not self.__eq__(other) @@ -58,23 +52,6 @@ class Requirements(OrderedDict): def __init__(self, *args): super(Requirements, self).__init__() - self.allow_dev = False - for v in args: - if isinstance(v, tuple): - override = private = dev = False - ref = v[0] - for elem in v[1:]: - if elem == "override": - override = True - elif elem == "private": - private = True - else: - raise ConanException("Unknown requirement config %s" % elem) - self.add(ref, private=private, override=override, dev=dev) - else: - self.add(v) - - def add_dev(self, *args): for v in args: if isinstance(v, tuple): override = private = False @@ -86,9 +63,9 @@ def add_dev(self, *args): private = True else: raise ConanException("Unknown requirement config %s" % elem) - self.add(ref, private=private, override=override, dev=True) + self.add(ref, private=private, override=override) else: - self.add(v, dev=True) + self.add(v) def copy(self): """ We need a custom copy as the normal one requires __init__ to be @@ -103,17 +80,15 @@ def copy(self): def iteritems(self): # FIXME: Just a trick to not change default testing conanfile for py3 return self.items() - def add(self, reference, private=False, override=False, dev=False): + def add(self, reference, private=False, override=False): """ to define requirements by the user in text, prior to any propagation """ assert isinstance(reference, six.string_types) - if dev and not self.allow_dev: - return conan_reference = ConanFileReference.loads(reference) name = conan_reference.name - new_requirement = Requirement(conan_reference, private, override, dev) + new_requirement = Requirement(conan_reference, private, override) old_requirement = self.get(name) if old_requirement and old_requirement != new_requirement: raise ConanException("Duplicated requirement %s != %s" @@ -138,7 +113,7 @@ def update(self, down_reqs, output, own_ref, down_ref): if own_ref: new_reqs.pop(own_ref.name, None) for name, req in self.items(): - if req.private or req.dev: + if req.private: continue if name in down_reqs: other_req = down_reqs[name] @@ -153,8 +128,8 @@ def update(self, down_reqs, output, own_ref, down_ref): new_reqs[name] = req return new_reqs - def __call__(self, conan_reference, private=False, override=False, dev=False): - self.add(conan_reference, private, override, dev) + def __call__(self, conan_reference, private=False, override=False): + self.add(conan_reference, private, override) def __repr__(self): result = [] diff --git a/conans/test/command/info_test.py b/conans/test/command/info_test.py index c6ecd1d0fc0..ad20a8ad9c4 100644 --- a/conans/test/command/info_test.py +++ b/conans/test/command/info_test.py @@ -294,15 +294,11 @@ def build_order_test(self): def diamond_build_order_test(self): self.client = TestClient() self._create("LibA", "0.1") - self._create("Dev1", "0.1") - self._create("LibE", "0.1", deps_dev=["Dev1/0.1@lasote/stable"]) + self._create("LibE", "0.1") self._create("LibF", "0.1") - self._create("LibG", "0.1") - self._create("Dev2", "0.1", deps=["LibG/0.1@lasote/stable"]) self._create("LibB", "0.1", ["LibA/0.1@lasote/stable", "LibE/0.1@lasote/stable"]) - self._create("LibC", "0.1", ["LibA/0.1@lasote/stable", "LibF/0.1@lasote/stable"], - deps_dev=["Dev2/0.1@lasote/stable"]) + self._create("LibC", "0.1", ["LibA/0.1@lasote/stable", "LibF/0.1@lasote/stable"]) self._create("LibD", "0.1", ["LibB/0.1@lasote/stable", "LibC/0.1@lasote/stable"], export=False) @@ -321,22 +317,15 @@ def diamond_build_order_test(self): self.client.user_io.out) self.client.run("info -bo=Dev1/0.1@lasote/stable") self.assertEqual("\n", self.client.user_io.out) - self.client.run("info --scope=LibE:dev=True -bo=Dev1/0.1@lasote/stable") - self.assertIn("[Dev1/0.1@lasote/stable], [LibE/0.1@lasote/stable], " - "[LibB/0.1@lasote/stable]", self.client.user_io.out) self.client.run("info -bo=LibG/0.1@lasote/stable") self.assertEqual("\n", self.client.user_io.out) - self.client.run("info --scope=LibC:dev=True -bo=LibG/0.1@lasote/stable") - self.assertIn("[LibG/0.1@lasote/stable], [Dev2/0.1@lasote/stable], " - "[LibC/0.1@lasote/stable]", self.client.user_io.out) self.client.run("info --build_order=ALL") self.assertIn("[LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, LibF/0.1@lasote/stable], " "[LibB/0.1@lasote/stable, LibC/0.1@lasote/stable]", self.client.user_io.out) - self.client.run("info --build_order=ALL --scope=ALL:dev=True") - self.assertIn("[Dev1/0.1@lasote/stable, LibG/0.1@lasote/stable], " - "[Dev2/0.1@lasote/stable, LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, " + self.client.run("info --build_order=ALL") + self.assertIn("[LibA/0.1@lasote/stable, LibE/0.1@lasote/stable, " "LibF/0.1@lasote/stable], [LibB/0.1@lasote/stable, LibC/0.1@lasote/stable]", self.client.user_io.out) diff --git a/conans/test/integration/conan_scopes_test.py b/conans/test/integration/conan_scopes_test.py index 835f1640ff7..cc99f97ae57 100644 --- a/conans/test/integration/conan_scopes_test.py +++ b/conans/test/integration/conan_scopes_test.py @@ -33,7 +33,7 @@ class HelloConan(ConanFile): version = "0.1" def config(self): if self.scope.other: - self.requires("Hello/0.1@lasote/stable", dev=True) + self.requires("Hello/0.1@lasote/stable") ''' files["conanfile.py"] = conanfile client.save(files, clean_first=True) @@ -215,49 +215,3 @@ def build(self): self.assertIn("WARN: CONFIG_CONSUMER OTHER", client.user_io.out) self.assertNotIn("WARN: BUILD_CONSUMER DEV", client.user_io.out) self.assertNotIn("WARN: BUILD_CONSUMER OTHER", client.user_io.out) - - def conan_dev_requires_test(self): - client = TestClient() - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - name = "Base" - version = "0.1" -''' - files = {} - files["conanfile.py"] = conanfile - client.save(files) - client.run("export lasote/stable") - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - dev_requires = "Base/0.1@lasote/stable" - name = "Hello" - version = "0.1" -''' - files = {} - files["conanfile.py"] = conanfile - client.save(files) - client.run("export lasote/stable") - conanfile = ''' -from conans import ConanFile - -class HelloConan(ConanFile): - dev_requires = "Hello/0.1@lasote/stable" - ''' - files["conanfile.py"] = conanfile - client.save(files, clean_first=True) - - client.run("install --build") - self.assertIn("Hello/0.1@lasote/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - client.user_io.out) - self.assertNotIn("Base/0.1@lasote/stable", client.user_io.out) - client.run("install --build -sc dev=False") - self.assertNotIn("Hello/0.1@lasote/stable", client.user_io.out) - self.assertNotIn("Base/0.1@lasote/stable", client.user_io.out) - client.run("install --build -sc dev=True -sc Hello:dev=True") - self.assertIn("Hello/0.1@lasote/stable:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9", - client.user_io.out) - self.assertIn("Base/0.1@lasote/stable", client.user_io.out) diff --git a/conans/test/model/options_test.py b/conans/test/model/options_test.py index 30105fcc150..652fba9553b 100644 --- a/conans/test/model/options_test.py +++ b/conans/test/model/options_test.py @@ -150,7 +150,7 @@ def test_dumps(self): "Poco:deps_bundled=True"])) def test_sha_constant(self): - self.assertEqual(self.sut.sha({"Boost", "Poco"}), + self.assertEqual(self.sut.sha, "2442d43f1d558621069a15ff5968535f818939b5") self.sut.new_option = False self.sut["Boost"].new_option = "off" @@ -165,5 +165,5 @@ def test_sha_constant(self): "Boost:thread.multi=off", "Poco:deps_bundled=True", "Poco:new_option=0"])) - self.assertEqual(self.sut.sha({"Boost", "Poco"}), + self.assertEqual(self.sut.sha, "2442d43f1d558621069a15ff5968535f818939b5") From 49b47789ca2706ceb4c6d5d802747a7436332ead Mon Sep 17 00:00:00 2001 From: Jonathan Newnham Date: Wed, 20 Sep 2017 00:43:41 +1000 Subject: [PATCH 37/39] Add include directories for the Resource compiler (#1770) visual_studio generator: Add include directories for the resource compiler, so that it can find and use *.h files that define resource symbols and IDs. This will allow dependencies to use resources from the projects they depend on. (use the same directories as normal header includes) This change should not break anyone because they will only need it if they are including header files, and if they are doing that, the header files should have custom names anyway. Related to conan-io #1421. --- conans/client/generators/visualstudio.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conans/client/generators/visualstudio.py b/conans/client/generators/visualstudio.py index ceaa0bef418..b92b0a77400 100644 --- a/conans/client/generators/visualstudio.py +++ b/conans/client/generators/visualstudio.py @@ -32,6 +32,9 @@ class VisualStudioGenerator(Generator): {include_dirs}%(AdditionalIncludeDirectories) + + {include_dirs}%(AdditionalIncludeDirectories) + ''' From 24c7866c74c59fd95ae7a4003a19f94f1b5ccff3 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Tue, 19 Sep 2017 16:46:00 +0200 Subject: [PATCH 38/39] Travis: Less builds if not release or master (#1769) * Try conditional builds * Try now * validated * Force build * Try with osx_image and language generic --- .travis.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3727ab30fca..c23914bc946 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,21 @@ -language: python -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 os: linux sudo: required dist: trusty +language: python matrix: include: + - python: 2.7 + - python: 3.4 + if: branch =~ (^release.*)|(^master) + - python: 3.5 + if: branch =~ (^release.*)|(^master) + - python: 3.6 - language: generic os: osx osx_image: xcode8.3 env: PYVER=py27 - + if: branch =~ (^release.*)|(^master) - language: generic os: osx osx_image: xcode8.3 @@ -25,6 +26,7 @@ install: - ./.ci/travis/install.sh before_script: - export PYTHONPATH=$PYTHONPATH:$(pwd) + # command to run tests script: - ulimit -n 2048 # Error with py3 and OSX, max file descriptors From ff405a0f60fd3672d0f5a4c39250015ba9edc573 Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Date: Tue, 19 Sep 2017 17:33:56 +0200 Subject: [PATCH 39/39] 0.27.0 --- conans/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/__init__.py b/conans/__init__.py index 495415e0810..fc9d70be955 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,4 +16,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.27.0-dev' +__version__ = '0.27.0'