From 22ff88d2652fe0a3782bb9caeb850d7b79aadc5c Mon Sep 17 00:00:00 2001 From: Chris Fischer Date: Tue, 3 Mar 2020 10:58:27 -0700 Subject: [PATCH] Squashed 'manage_externals/' changes from 1926530..fde04e4 fde04e4 Merge pull request #138 from billsacks/add_python38_tests 37e4c4a Do not update dictionary in-place in loop 7e8474b Remove testing on mac os 7f41c56 Fix pylint issue 3065b0d Add travis-ci tests with python3.7 and python3.8 34fbf55 Add support for git sparse checkout 6c6ef9f Fix pylint errors 6a659ad Added test for sparse checkout and updated documentation 1443243 Support for git sparsecheckout via read-tree. a48558d Merge pull request #119 from gold2718/submodules f72ffe7 Do not try git submodule update if no .gitmodules file (git bug) 804e0af Fix a pylint error 45aef95 Addressed review concerns 7da5031 New capability to use git submodule information to checkout externals git-subtree-dir: manage_externals git-subtree-split: fde04e4d9a758b3aa277aa5fa44a59f5153f2958 --- .travis.yml | 17 +- README.md | 17 +- manic/checkout.py | 15 + manic/externals_description.py | 359 ++++++++++++++-- manic/repository.py | 20 +- manic/repository_git.py | 124 +++++- manic/repository_svn.py | 13 +- manic/sourcetree.py | 75 +++- .../14/2711fdbbcb8034d7cad6bae6801887b12fe61d | Bin 0 -> 83 bytes .../60/7ec299c17dd285c029edc41a0109e49d441380 | Bin 0 -> 168 bytes .../b7/692b6d391899680da7b9b6fd8af4c413f06fe7 | Bin 0 -> 137 bytes .../d1/163870d19c3dee34fada3a76b785cfa2a8424b | Bin 0 -> 130 bytes .../d8/ed2f33179d751937f8fde2e33921e4827babf4 | Bin 0 -> 60 bytes test/repos/simple-ext.git/refs/heads/master | 2 +- test/repos/simple-ext.git/refs/tags/tag2 | 1 + test/test_sys_checkout.py | 393 +++++++++++++++++- test/test_unit_externals_description.py | 12 +- test/test_unit_repository.py | 25 +- test/test_unit_repository_git.py | 3 +- 19 files changed, 947 insertions(+), 129 deletions(-) create mode 100644 test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d create mode 100644 test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 create mode 100644 test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 create mode 100644 test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b create mode 100644 test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 create mode 100644 test/repos/simple-ext.git/refs/tags/tag2 diff --git a/.travis.yml b/.travis.yml index b32f81bd28..1990cb9604 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,3 @@ -# NOTE(bja, 2017-11) travis-ci dosen't support python language builds -# on mac os. As a work around, we use built-in python on linux, and -# declare osx a 'generic' language, and create our own python env. - language: python os: linux python: @@ -9,17 +5,8 @@ python: - "3.4" - "3.5" - "3.6" -matrix: - include: - - os: osx - language: generic - before_install: - # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough! - # - brew update - # - brew outdated python2 || brew upgrade python2 - - pip install virtualenv - - virtualenv env -p python2 - - source env/bin/activate + - "3.7" + - "3.8" install: - pip install -r test/requirements.txt before_script: diff --git a/README.md b/README.md index 15e45ffb71..c931c8e213 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. description file: $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --excernals my-externals.cfg + $ ./manage_externals/checkout_externals --externals my-externals.cfg * Status summary of the repositories managed by checkout_externals: @@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + * Lines begining with '#' or ';' are comments and will be ignored. # Obtaining this tool, reporting issues, etc. diff --git a/manic/checkout.py b/manic/checkout.py index afd3a27886..edc5655954 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -227,6 +227,21 @@ def commandline_arguments(args=None): Now, %(prog)s will process Externals.cfg and also process Externals_LIBX.cfg as if it was a sub-external. + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + * Lines beginning with '#' or ';' are comments and will be ignored. # Obtaining this tool, reporting issues, etc. diff --git a/manic/externals_description.py b/manic/externals_description.py index b32d37cfc6..b0c4f736a7 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -22,15 +22,17 @@ import os.path import re -# ConfigParser was renamed in python2 to configparser. In python2, -# ConfigParser returns byte strings, str, instead of unicode. We need -# unicode to be compatible with xml and json parser and python3. +# ConfigParser in python2 was renamed to configparser in python3. +# In python2, ConfigParser returns byte strings, str, instead of unicode. +# We need unicode to be compatible with xml and json parser and python3. try: # python2 from ConfigParser import SafeConfigParser as config_parser from ConfigParser import MissingSectionHeaderError from ConfigParser import NoSectionError, NoOptionError + USE_PYTHON2 = True + def config_string_cleaner(text): """convert strings into unicode """ @@ -41,6 +43,8 @@ def config_string_cleaner(text): from configparser import MissingSectionHeaderError from configparser import NoSectionError, NoOptionError + USE_PYTHON2 = False + def config_string_cleaner(text): """Python3 already uses unicode strings, so just return the string without modification. @@ -49,6 +53,7 @@ def config_string_cleaner(text): return text from .utils import printlog, fatal_error, str_to_bool, expand_local_url +from .utils import execute_subprocess from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR # @@ -59,8 +64,8 @@ def config_string_cleaner(text): def read_externals_description_file(root_dir, file_name): - """Given a file name containing a externals description, determine the - format and read it into it's internal representation. + """Read a file containing an externals description and + create its internal representation. """ root_dir = os.path.abspath(root_dir) @@ -70,29 +75,193 @@ def read_externals_description_file(root_dir, file_name): file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): - msg = ('ERROR: Model description file, "{0}", does not ' - 'exist at path:\n {1}\nDid you run from the root of ' - 'the source tree?'.format(file_name, file_path)) + if file_name.lower() == "none": + msg = ('INTERNAL ERROR: Attempt to read externals file ' + 'from {0} when not configured'.format(file_path)) + else: + msg = ('ERROR: Model description file, "{0}", does not ' + 'exist at path:\n {1}\nDid you run from the root of ' + 'the source tree?'.format(file_name, file_path)) + fatal_error(msg) + externals_description = None + if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: + externals_description = read_gitmodules_file(root_dir, file_name) + else: + try: + config = config_parser() + config.read(file_path) + externals_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if externals_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + + return externals_description + +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + def __init__(self, filename): + with open(filename, 'r') as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = '' + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() + +def git_submodule_status(repo_dir): + """Run the git submodule status command to obtain submodule hashes. + """ + # This function is here instead of GitRepository to avoid a dependency loop + cwd = os.getcwd() + os.chdir(repo_dir) + cmd = ['git', 'submodule', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + submodules = {} + submods = git_output.split('\n') + for submod in submods: + if submod: + status = submod[0] + items = submod[1:].split(' ') + if len(items) > 2: + tag = items[2] + else: + tag = None + + submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} + + os.chdir(cwd) + return submodules + +def parse_submodules_desc_section(section_items, file_path): + """Find the path and url for this submodule description""" + path = None + url = None + for item in section_items: + name = item[0].strip().lower() + if name == 'path': + path = item[1].strip() + elif name == 'url': + url = item[1].strip() + else: + msg = 'WARNING: Ignoring unknown {} property, in {}' + msg = msg.format(item[0], file_path) # fool pylint + logging.warning(msg) + + return path, url + +def read_gitmodules_file(root_dir, file_name): + # pylint: disable=deprecated-method + # Disabling this check because the method is only used for python2 + """Read a .gitmodules file and convert it to be compatible with an + externals description. + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing submodules description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + msg = ('ERROR: submodules description file, "{0}", does not ' + 'exist at path:\n {1}'.format(file_name, file_path)) + fatal_error(msg) + + submodules_description = None externals_description = None try: config = config_parser() - config.read(file_path) - externals_description = config + if USE_PYTHON2: + config.readfp(LstripReader(file_path), filename=file_name) + else: + config.read_file(LstripReader(file_path), source=file_name) + + submodules_description = config except MissingSectionHeaderError: # not a cfg file pass - if externals_description is None: + if submodules_description is None: msg = 'Unknown file format!' fatal_error(msg) + else: + # Convert the submodules description to an externals description + externals_description = config_parser() + # We need to grab all the commit hashes for this repo + submods = git_submodule_status(root_dir) + for section in submodules_description.sections(): + if section[0:9] == 'submodule': + sec_name = section[9:].strip(' "') + externals_description.add_section(sec_name) + section_items = submodules_description.items(section) + path, url = parse_submodules_desc_section(section_items, + file_path) + + if path is None: + msg = 'Submodule {} missing path'.format(sec_name) + fatal_error(msg) - return externals_description + if url is None: + msg = 'Submodule {} missing url'.format(sec_name) + fatal_error(msg) + externals_description.set(sec_name, + ExternalsDescription.PATH, path) + externals_description.set(sec_name, + ExternalsDescription.PROTOCOL, 'git') + externals_description.set(sec_name, + ExternalsDescription.REPO_URL, url) + externals_description.set(sec_name, + ExternalsDescription.REQUIRED, 'True') + git_hash = submods[sec_name]['hash'] + externals_description.set(sec_name, + ExternalsDescription.HASH, git_hash) + + # Required items + externals_description.add_section(DESCRIPTION_SECTION) + externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + + return externals_description def create_externals_description( - model_data, model_format='cfg', components=None): + model_data, model_format='cfg', components=None, parent_repo=None): """Create the a externals description object from the provided data """ externals_description = None @@ -103,7 +272,7 @@ def create_externals_description( major, _, _ = get_cfg_schema_version(model_data) if major == 1: externals_description = ExternalsDescriptionConfigV1( - model_data, components=components) + model_data, components=components, parent_repo=parent_repo) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -173,18 +342,21 @@ class ExternalsDescription(dict): # keywords defining the interface into the externals description data EXTERNALS = 'externals' BRANCH = 'branch' - REPO = 'repo' - REQUIRED = 'required' - TAG = 'tag' + SUBMODULE = 'from_submodule' + HASH = 'hash' + NAME = 'name' PATH = 'local_path' PROTOCOL = 'protocol' + REPO = 'repo' REPO_URL = 'repo_url' - HASH = 'hash' - NAME = 'name' + REQUIRED = 'required' + TAG = 'tag' + SPARSE = 'sparse' PROTOCOL_EXTERNALS_ONLY = 'externals_only' PROTOCOL_GIT = 'git' PROTOCOL_SVN = 'svn' + GIT_SUBMODULES_FILENAME = '.gitmodules' KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] # v1 xml keywords @@ -197,15 +369,17 @@ class ExternalsDescription(dict): _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', + SUBMODULE : True, REPO: {PROTOCOL: 'string', REPO_URL: 'string', TAG: 'string', BRANCH: 'string', HASH: 'string', - } - } + SPARSE: 'string', + } + } - def __init__(self): + def __init__(self, parent_repo=None): """Convert the xml into a standardized dict that can be used to construct the source objects @@ -218,6 +392,7 @@ def __init__(self): self._input_major = None self._input_minor = None self._input_patch = None + self._parent_repo = parent_repo def _verify_schema_version(self): """Use semantic versioning rules to verify we can process this schema. @@ -265,6 +440,7 @@ def _check_user_input(self): self._validate() def _check_data(self): + # pylint: disable=too-many-branches,too-many-statements """Check user supplied data is valid where possible. """ for ext_name in self.keys(): @@ -282,6 +458,13 @@ def _check_data(self): ext_name)) fatal_error(msg) + if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) + and (self.SUBMODULE in self[ext_name])): + msg = ('self.SUBMODULE is only supported with {0} protocol, ' + '"{1}" is defined as an {2} repository') + fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, + self[ext_name][self.REPO][self.PROTOCOL])) + if (self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_EXTERNALS_ONLY): ref_count = 0 @@ -301,11 +484,23 @@ def _check_data(self): found_refs = '"{0} = {1}", {2}'.format( self.HASH, self[ext_name][self.REPO][self.HASH], found_refs) + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.SUBMODULE, + self[ext_name][self.SUBMODULE], found_refs) if ref_count > 1: - msg = ('Model description is over specified! Only one of ' - '"tag", "branch", or "hash" may be specified for ' - 'repo description of "{0}".'.format(ext_name)) + msg = 'Model description is over specified! ' + if self.SUBMODULE in self[ext_name]: + msg += ('from_submodule is not compatible with ' + '"tag", "branch", or "hash" ') + else: + msg += (' Only one of "tag", "branch", or "hash" ' + 'may be specified ') + + msg += 'for repo description of "{0}".'.format(ext_name) msg = '{0}\nFound: {1}'.format(msg, found_refs) fatal_error(msg) elif ref_count < 1: @@ -314,17 +509,39 @@ def _check_data(self): 'repo description of "{0}"'.format(ext_name)) fatal_error(msg) - if self.REPO_URL not in self[ext_name][self.REPO]: + if (self.REPO_URL not in self[ext_name][self.REPO] and + (self.SUBMODULE not in self[ext_name] or + not self[ext_name][self.SUBMODULE])): msg = ('Model description is under specified! Must have ' '"repo_url" in repo ' 'description for "{0}"'.format(ext_name)) fatal_error(msg) - url = expand_local_url( - self[ext_name][self.REPO][self.REPO_URL], ext_name) - self[ext_name][self.REPO][self.REPO_URL] = url + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + if self.REPO_URL in self[ext_name][self.REPO]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible ' + 'with {0} keyword for'.format(self.REPO_URL)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.PATH in self[ext_name]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible with ' + '{0} keyword for'.format(self.PATH)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.REPO_URL in self[ext_name][self.REPO]: + url = expand_local_url( + self[ext_name][self.REPO][self.REPO_URL], ext_name) + self[ext_name][self.REPO][self.REPO_URL] = url def _check_optional(self): + # pylint: disable=too-many-branches """Some fields like externals, repo:tag repo:branch are (conditionally) optional. We don't want the user to be required to enter them in every externals description file, but @@ -332,6 +549,7 @@ def _check_optional(self): default values if appropriate. """ + submod_desc = None # Only load submodules info once for field in self: # truely optional if self.EXTERNALS not in self[field]: @@ -346,6 +564,72 @@ def _check_optional(self): self[field][self.REPO][self.HASH] = EMPTY_STR if self.REPO_URL not in self[field][self.REPO]: self[field][self.REPO][self.REPO_URL] = EMPTY_STR + if self.SPARSE not in self[field][self.REPO]: + self[field][self.REPO][self.SPARSE] = EMPTY_STR + + # from_submodule has a complex relationship with other fields + if self.SUBMODULE in self[field]: + # User wants to use submodule information, is it available? + if self._parent_repo is None: + # No parent == no submodule information + PPRINTER.pprint(self[field]) + msg = 'No parent submodule for "{0}"'.format(field) + fatal_error(msg) + elif self._parent_repo.protocol() != self.PROTOCOL_GIT: + PPRINTER.pprint(self[field]) + msg = 'Parent protocol, "{0}", does not support submodules' + fatal_error(msg.format(self._parent_repo.protocol())) + else: + args = self._repo_config_from_submodule(field, submod_desc) + repo_url, repo_path, ref_hash, submod_desc = args + + if repo_url is None: + msg = ('Cannot checkout "{0}" as a submodule, ' + 'repo not found in {1} file') + fatal_error(msg.format(field, + self.GIT_SUBMODULES_FILENAME)) + # Fill in submodule fields + self[field][self.REPO][self.REPO_URL] = repo_url + self[field][self.REPO][self.HASH] = ref_hash + self[field][self.PATH] = repo_path + + if self[field][self.SUBMODULE]: + # We should get everything from the parent submodule + # configuration. + pass + # No else (from _submodule = False is the default) + else: + # Add the default value (not using submodule information) + self[field][self.SUBMODULE] = False + + def _repo_config_from_submodule(self, field, submod_desc): + """Find the external config information for a repository from + its submodule configuration information. + """ + if submod_desc is None: + repo_path = os.getcwd() # Is this always correct? + submod_file = self._parent_repo.submodules_file(repo_path=repo_path) + if submod_file is None: + msg = ('Cannot checkout "{0}" from submodule information\n' + ' Parent repo, "{1}" does not have submodules') + fatal_error(msg.format(field, self._parent_repo.name())) + + submod_file = read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_file) + + # Can we find our external? + repo_url = None + repo_path = None + ref_hash = None + for ext_field in submod_desc: + if field == ext_field: + ext = submod_desc[ext_field] + repo_url = ext[self.REPO][self.REPO_URL] + repo_path = ext[self.PATH] + ref_hash = ext[self.REPO][self.HASH] + break + + return repo_url, repo_path, ref_hash, submod_desc def _validate(self): """Validate that the parsed externals description contains all necessary @@ -383,11 +667,12 @@ def validate_data_struct(schema, data): if isinstance(schema, dict) and isinstance(data, dict): # Both are dicts, recursively verify that all fields # in schema are present in the data. - for k in schema: - in_ref = in_ref and (k in data) + for key in schema: + in_ref = in_ref and (key in data) if in_ref: valid = valid and ( - validate_data_struct(schema[k], data[k])) + validate_data_struct(schema[key], data[key])) + is_valid = in_ref and valid else: # non-recursive structure. verify data and schema have @@ -434,9 +719,9 @@ def __init__(self, model_data, components=None): self._input_patch = 0 self._verify_schema_version() if components: - for k in model_data.items(): - if k not in components: - del model_data[k] + for key in model_data.items(): + if key not in components: + del model_data[key] self.update(model_data) self._check_user_input() @@ -448,12 +733,12 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data, components=None): + def __init__(self, model_data, components=None, parent_repo=None): """Convert the config data into a standardized dict that can be used to construct the source objects """ - ExternalsDescription.__init__(self) + ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 self._schema_minor = 1 self._schema_patch = 0 diff --git a/manic/repository.py b/manic/repository.py index d01849d37a..ea4230fb7b 100644 --- a/manic/repository.py +++ b/manic/repository.py @@ -21,6 +21,7 @@ def __init__(self, component_name, repo): self._branch = repo[ExternalsDescription.BRANCH] self._hash = repo[ExternalsDescription.HASH] self._url = repo[ExternalsDescription.REPO_URL] + self._sparse = repo[ExternalsDescription.SPARSE] if self._url is EMPTY_STR: fatal_error('repo must have a URL') @@ -40,12 +41,14 @@ def __init__(self, component_name, repo): fatal_error('repo {0} must have exactly one of ' 'tag, branch or hash.'.format(self._name)) - def checkout(self, base_dir_path, repo_dir_name, verbosity): # pylint: disable=unused-argument + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the source. If the repo destination directory does not exist, checkout the correce branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) """ msg = ('DEV_ERROR: checkout method must be implemented in all ' 'repository classes! {0}'.format(self.__class__.__name__)) @@ -59,6 +62,11 @@ def status(self, stat, repo_dir_path): # pylint: disable=unused-argument 'repository classes! {0}'.format(self.__class__.__name__)) fatal_error(msg) + def submodules_file(self, repo_path=None): + # pylint: disable=no-self-use,unused-argument + """Stub for use by non-git VC systems""" + return None + def url(self): """Public access of repo url. """ @@ -78,3 +86,13 @@ def hash(self): """Public access of repo hash. """ return self._hash + + def name(self): + """Public access of repo name. + """ + return self._name + + def protocol(self): + """Public access of repo protocol. + """ + return self._protocol diff --git a/manic/repository_git.py b/manic/repository_git.py index efb775d0bc..f986051001 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -12,6 +12,7 @@ from .global_constants import VERBOSITY_VERBOSE from .repository import Repository from .externals_status import ExternalStatus +from .externals_description import ExternalsDescription, git_submodule_status from .utils import expand_local_url, split_remote_url, is_remote_url from .utils import fatal_error, printlog from .utils import execute_subprocess @@ -41,17 +42,19 @@ def __init__(self, component_name, repo): Parse repo (a XML element). """ Repository.__init__(self, component_name, repo) + self._gitmodules = None + self._submods = None # ---------------------------------------------------------------- # # Public API, defined by Repository # # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correce + If the repo destination directory does not exist, checkout the correct branch or tag. """ repo_dir_path = os.path.join(base_dir_path, repo_dir_name) @@ -59,7 +62,15 @@ def checkout(self, base_dir_path, repo_dir_name, verbosity): if (repo_dir_exists and not os.listdir( repo_dir_path)) or not repo_dir_exists: self._clone_repo(base_dir_path, repo_dir_name, verbosity) - self._checkout_ref(repo_dir_path, verbosity) + self._checkout_ref(repo_dir_path, verbosity, recursive) + gmpath = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_dir_path) + else: + self._gitmodules = None + self._submods = None def status(self, stat, repo_dir_path): """ @@ -72,6 +83,16 @@ def status(self, stat, repo_dir_path): if os.path.exists(repo_dir_path): self._status_summary(stat, repo_dir_path) + def submodules_file(self, repo_path=None): + if repo_path is not None: + gmpath = os.path.join(repo_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_path) + + return self._gitmodules + # ---------------------------------------------------------------- # # Internal work functions @@ -282,23 +303,30 @@ def _create_remote_name(self): remote_name = "{0}_{1}".format(base_name, repo_name) return remote_name - def _checkout_ref(self, repo_dir, verbosity): + def _checkout_ref(self, repo_dir, verbosity, submodules): """Checkout the user supplied reference + if is True, recursively initialize and update + the repo's submodules """ # import pdb; pdb.set_trace() cwd = os.getcwd() os.chdir(repo_dir) if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity) + self._checkout_local_ref(verbosity, submodules) else: - self._checkout_external_ref(verbosity) + self._checkout_external_ref(verbosity, submodules) + + if self._sparse: + self._sparse_checkout(repo_dir, verbosity) os.chdir(cwd) - def _checkout_local_ref(self, verbosity): + + def _checkout_local_ref(self, verbosity, submodules): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when checkout out the ref. - + if is True, recursively initialize and update + the repo's submodules """ if self._tag: ref = self._tag @@ -308,10 +336,12 @@ def _checkout_local_ref(self, verbosity): ref = self._hash self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity) + self._git_checkout_ref(ref, verbosity, submodules) - def _checkout_external_ref(self, verbosity): + def _checkout_external_ref(self, verbosity, submodules): """Checkout the reference from a remote repository + if is True, recursively initialize and update + the repo's submodules """ if self._tag: ref = self._tag @@ -326,14 +356,28 @@ def _checkout_external_ref(self, verbosity): self._git_remote_add(remote_name, self._url) self._git_fetch(remote_name) - # NOTE(bja, 2018-03) we need to send seperate ref and remote + # NOTE(bja, 2018-03) we need to send separate ref and remote # name to check_for_vaild_ref, but the combined name to # checkout_ref! self._check_for_valid_ref(ref, remote_name) if self._branch: ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity) + self._git_checkout_ref(ref, verbosity, submodules) + + def _sparse_checkout(self, repo_dir, verbosity): + """Use git read-tree to thin the working tree.""" + cwd = os.getcwd() + + cmd = ['cp', self._sparse, os.path.join(repo_dir, + '.git/info/sparse-checkout')] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + os.chdir(repo_dir) + self._git_sparse_checkout(verbosity) + + os.chdir(cwd) def _check_for_valid_ref(self, ref, remote_name=None): """Try some basic sanity checks on the user supplied reference so we @@ -687,6 +731,19 @@ def _git_remote_verbose(): git_output = execute_subprocess(cmd, output_to_caller=True) return git_output + @staticmethod + def has_submodules(repo_dir_path=None): + """Return True iff the repository at (or the current + directory if is None) has a '.gitmodules' file + """ + if repo_dir_path is None: + fname = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + + return os.path.exists(fname) + # ---------------------------------------------------------------- # # system call to git for sideffects modifying the working tree @@ -696,28 +753,34 @@ def _git_remote_verbose(): def _git_clone(url, repo_dir_name, verbosity): """Run git clone for the side effect of creating a repository. """ - cmd = ['git', 'clone', '--quiet', url, repo_dir_name] + cmd = ['git', 'clone', '--quiet'] + subcmd = None + + cmd.extend([url, repo_dir_name]) if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) + if subcmd is not None: + os.chdir(repo_dir_name) + execute_subprocess(subcmd) @staticmethod def _git_remote_add(name, url): - """Run the git remote command to for the side effect of adding a remote + """Run the git remote command for the side effect of adding a remote """ cmd = ['git', 'remote', 'add', name, url] execute_subprocess(cmd) @staticmethod def _git_fetch(remote_name): - """Run the git fetch command to for the side effect of updating the repo + """Run the git fetch command for the side effect of updating the repo """ cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] execute_subprocess(cmd) @staticmethod - def _git_checkout_ref(ref, verbosity): - """Run the git checkout command to for the side effect of updating the repo + def _git_checkout_ref(ref, verbosity, submodules): + """Run the git checkout command for the side effect of updating the repo Param: ref is a reference to a local or remote object in the form 'origin/my_feature', or 'tag1'. @@ -727,3 +790,30 @@ def _git_checkout_ref(ref, verbosity): if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) + if submodules: + GitRepository._git_update_submodules(verbosity) + + @staticmethod + def _git_sparse_checkout(verbosity): + """Configure repo via read-tree.""" + cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + cmd = ['git', 'read-tree', '-mu', 'HEAD'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + + @staticmethod + def _git_update_submodules(verbosity): + """Run git submodule update for the side effect of updating this + repo's submodules. + """ + # First, verify that we have a .gitmodules file + if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): + cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + + execute_subprocess(cmd) diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 90e37344b2..408ed84676 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -56,7 +56,7 @@ def __init__(self, component_name, repo, ignore_ancestry=False): # Public API, defined by Repository # # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument """Checkout or update the working copy If the repo destination directory exists, switch the sandbox to @@ -64,6 +64,8 @@ def checkout(self, base_dir_path, repo_dir_name, verbosity): If the repo destination directory does not exist, checkout the correct branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) """ repo_dir_path = os.path.join(base_dir_path, repo_dir_name) @@ -138,9 +140,7 @@ def _abort_if_dirty(self, repo_dir_path, message): To recover: Clean up the above directory (resolving conflicts, etc.), then rerun checkout_externals. -""".format(cwd=repo_dir_path, - message=message, - status=status) +""".format(cwd=repo_dir_path, message=message, status=status) fatal_error(errmsg) @@ -220,9 +220,8 @@ def xml_status_is_dirty(svn_output): continue if item == SVN_UNVERSIONED: continue - else: - is_dirty = True - break + is_dirty = True + break return is_dirty # ---------------------------------------------------------------- diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 0479db1a79..b9c9c21082 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -11,12 +11,12 @@ from .externals_description import read_externals_description_file from .externals_description import create_externals_description from .repository_factory import create_repository +from .repository_git import GitRepository from .externals_status import ExternalStatus from .utils import fatal_error, printlog from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR from .global_constants import VERBOSITY_VERBOSE - class _External(object): """ _External represents an external object inside a SourceTree @@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): self._externals = EMPTY_STR self._externals_sourcetree = None self._stat = ExternalStatus() + self._sparse = None # Parse the sub-elements # _path : local path relative to the containing source tree @@ -61,14 +62,20 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): self._required = ext_description[ExternalsDescription.REQUIRED] self._externals = ext_description[ExternalsDescription.EXTERNALS] - if self._externals: - self._create_externals_sourcetree() + # Treat a .gitmodules file as a backup externals config + if not self._externals: + if GitRepository.has_submodules(self._repo_dir_path): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + repo = create_repository( name, ext_description[ExternalsDescription.REPO], svn_ignore_ancestry=svn_ignore_ancestry) if repo: self._repo = repo + if self._externals and (self._externals.lower() != 'none'): + self._create_externals_sourcetree() + def get_name(self): """ Return the external object's name @@ -125,7 +132,7 @@ def status(self): if self._externals and self._externals_sourcetree: # we expect externals and they exist cwd = os.getcwd() - # SourceTree expecteds to be called from the correct + # SourceTree expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) ext_stats = self._externals_sourcetree.status(self._local_path) @@ -148,7 +155,7 @@ def checkout(self, verbosity, load_all): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce + If the repo destination directory does not exist, checkout the correct branch or tag. If load_all is True, also load all of the the externals sub-externals. """ @@ -183,13 +190,14 @@ def checkout(self, verbosity, load_all): checkout_verbosity = verbosity - 1 else: checkout_verbosity = verbosity - self._repo.checkout(self._base_dir_path, - self._repo_dir_name, checkout_verbosity) + + self._repo.checkout(self._base_dir_path, self._repo_dir_name, + checkout_verbosity, self.clone_recursive()) def checkout_externals(self, verbosity, load_all): """Checkout the sub-externals for this object """ - if self._externals: + if self.load_externals(): if self._externals_sourcetree: # NOTE(bja, 2018-02): the subtree externals objects # were created during initial status check. Updating @@ -201,6 +209,24 @@ def checkout_externals(self, verbosity, load_all): self._create_externals_sourcetree() self._externals_sourcetree.checkout(verbosity, load_all) + def load_externals(self): + 'Return True iff an externals file should be loaded' + load_ex = False + if os.path.exists(self._repo_dir_path): + if self._externals: + if self._externals.lower() != 'none': + load_ex = os.path.exists(os.path.join(self._repo_dir_path, + self._externals)) + + return load_ex + + def clone_recursive(self): + 'Return True iff any .gitmodules files should be processed' + # Try recursive unless there is an externals entry + recursive = not self._externals + + return recursive + def _create_externals_sourcetree(self): """ """ @@ -213,6 +239,15 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) + if self._externals.lower() == 'none': + msg = ('Internal: Attempt to create source tree for ' + 'externals = none in {}'.format(self._repo_dir_path)) + fatal_error(msg) + + if not os.path.exists(self._externals): + if GitRepository.has_submodules(): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + if not os.path.exists(self._externals): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! @@ -224,11 +259,11 @@ def _create_externals_sourcetree(self): externals_root = self._repo_dir_path model_data = read_externals_description_file(externals_root, self._externals) - externals = create_externals_description(model_data) + externals = create_externals_description(model_data, + parent_repo=self._repo) self._externals_sourcetree = SourceTree(externals_root, externals) os.chdir(cwd) - class SourceTree(object): """ SourceTree represents a group of managed externals @@ -264,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): for comp in load_comps: printlog('{0}, '.format(comp), end='') stat = self._all_components[comp].status() + stat_final = {} for name in stat.keys(): # check if we need to append the relative_path_base to # the path so it will be sorted in the correct order. - if not stat[name].path.startswith(relative_path_base): - stat[name].path = os.path.join(relative_path_base, - stat[name].path) - # store under key = updated path, and delete the - # old key. - comp_stat = stat[name] - del stat[name] - stat[comp_stat.path] = comp_stat - summary.update(stat) + if stat[name].path.startswith(relative_path_base): + # use as is, without any changes to path + stat_final[name] = stat[name] + else: + # append relative_path_base to path and store under key = updated path + modified_path = os.path.join(relative_path_base, + stat[name].path) + stat_final[modified_path] = stat[name] + stat_final[modified_path].path = modified_path + summary.update(stat_final) return summary diff --git a/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d b/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d new file mode 100644 index 0000000000000000000000000000000000000000..acaf7889b47c54ee0dea121c73d505ca14ad369b GIT binary patch literal 83 zcmV-Z0IdIb0ZYosPg1ZjWC+Q~ELKR%%t=)!&d4v#Nl{3x$Sf{V$jnnnRLILO%1z8s pNX|%2&dx6_QAh$}pz8eG%#xDS6o{JQg2bZYRJa;FE&z4gA7ySEC>H<# literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 b/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 new file mode 100644 index 0000000000000000000000000000000000000000..3f6959cc54afd45fa4f64be25ccb3bb8580183c9 GIT binary patch literal 168 zcmV;Z09XHb0hNwR4#F@D1zG15z5poBzY;=l6vuV}(Nff=3vN&02JB|>MsIi;$9n_k z!>-M$Ac)DAYy~^^qUu9WLY{J}xkT>CQ3)XSx>0V^p=O;s>7G-EI{FfcPQQP4}zEXhpI%P&f0aFl&|Gw+GS!K3kZ)1Ezh zejs~i1S3>cQEFmJZmM2MMG3KlvCEtNF?@%PbVOT{Nm)vLb%0Bl_``r7C@umAu6 literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 b/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 new file mode 100644 index 0000000000000000000000000000000000000000..f08ae820c9c89927f9898c5646134f7c519a6b04 GIT binary patch literal 60 zcmV-C0K@-y0V^p=O;s>4W-v4`Ff%bxC@xJ($t;Rb%gjmDE2$`95K$NWyZdy5$@Np$ Sc0Fs5Xy2&+OcnsW_!K#a0~pW% literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/refs/heads/master b/test/repos/simple-ext.git/refs/heads/master index 5c67504966..adf1ccb002 100644 --- a/test/repos/simple-ext.git/refs/heads/master +++ b/test/repos/simple-ext.git/refs/heads/master @@ -1 +1 @@ -9b75494003deca69527bb64bcaa352e801611dd2 +607ec299c17dd285c029edc41a0109e49d441380 diff --git a/test/repos/simple-ext.git/refs/tags/tag2 b/test/repos/simple-ext.git/refs/tags/tag2 new file mode 100644 index 0000000000..4160b6c494 --- /dev/null +++ b/test/repos/simple-ext.git/refs/tags/tag2 @@ -0,0 +1 @@ +b7692b6d391899680da7b9b6fd8af4c413f06fe7 diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 9ebfb0aeee..df726f2b70 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -42,6 +42,7 @@ from manic.externals_description import ExternalsDescription from manic.externals_description import DESCRIPTION_SECTION, VERSION_ITEM +from manic.externals_description import git_submodule_status from manic.externals_status import ExternalStatus from manic.repository_git import GitRepository from manic.utils import printlog, execute_subprocess @@ -87,6 +88,8 @@ SVN_TEST_REPO = 'https://github.com/escomp/cesm' +# Disable too-many-public-methods error +# pylint: disable=R0904 def setUpModule(): # pylint: disable=C0103 """Setup for all tests in this module. It is called once per module! @@ -139,7 +142,7 @@ def container_full(self, dest_dir): self.create_section(MIXED_REPO_NAME, 'mixed_req', branch='master', externals=CFG_SUB_NAME) - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_required(self, dest_dir): """Create a container externals file with only simple externals. @@ -155,7 +158,7 @@ def container_simple_required(self, dest_dir): self.create_section(SIMPLE_REPO_NAME, 'simp_hash', ref_hash='60b1cc1a38d63') - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_optional(self, dest_dir): """Create a container externals file with optional simple externals @@ -168,7 +171,7 @@ def container_simple_optional(self, dest_dir): self.create_section(SIMPLE_REPO_NAME, 'simp_opt', tag='tag1', required=False) - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_svn(self, dest_dir): """Create a container externals file with only simple externals. @@ -180,7 +183,26 @@ def container_simple_svn(self, dest_dir): self.create_svn_external('svn_branch', branch='trunk') self.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') - self._write_config(dest_dir) + self.write_config(dest_dir) + + def container_sparse(self, dest_dir): + """Create a container with a full external and a sparse external + + """ + # Create a file for a sparse pattern match + sparse_filename = 'sparse_checkout' + with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: + sfile.write('readme.txt') + + self.create_config() + self.create_section(SIMPLE_REPO_NAME, 'simp_tag', + tag='tag2') + + sparse_relpath = '../../{}'.format(sparse_filename) + self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self.write_config(dest_dir) def mixed_simple_base(self, dest_dir): """Create a mixed-use base externals file with only simple externals. @@ -197,7 +219,7 @@ def mixed_simple_base(self, dest_dir): self.create_section(SIMPLE_REPO_NAME, 'simp_hash', ref_hash='60b1cc1a38d63') - self._write_config(dest_dir) + self.write_config(dest_dir) def mixed_simple_sub(self, dest_dir): """Create a mixed-use sub externals file with only simple externals. @@ -211,9 +233,9 @@ def mixed_simple_sub(self, dest_dir): branch=REMOTE_BRANCH_FEATURE2, path=SUB_EXTERNALS_PATH) - self._write_config(dest_dir, filename=CFG_SUB_NAME) + self.write_config(dest_dir, filename=CFG_SUB_NAME) - def _write_config(self, dest_dir, filename=CFG_NAME): + def write_config(self, dest_dir, filename=CFG_NAME): """Write the configuration file to disk """ @@ -237,22 +259,41 @@ def create_metadata(self): self._schema_version) def create_section(self, repo_type, name, tag='', branch='', - ref_hash='', - required=True, path=EXTERNALS_NAME, externals=''): + ref_hash='', required=True, path=EXTERNALS_NAME, + externals='', repo_path=None, from_submodule=False, + sparse=''): + # pylint: disable=too-many-branches """Create a config section with autofilling some items and handling optional items. """ # pylint: disable=R0913 self._config.add_section(name) - self._config.set(name, ExternalsDescription.PATH, - os.path.join(path, name)) + if not from_submodule: + self._config.set(name, ExternalsDescription.PATH, + os.path.join(path, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_GIT) - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) - self._config.set(name, ExternalsDescription.REPO_URL, repo_url) + # from_submodules is incompatible with some other options, turn them off + if (from_submodule and + ((repo_path is not None) or tag or ref_hash or branch)): + printlog('create_section: "from_submodule" is incompatible with ' + '"repo_url", "tag", "hash", and "branch" options;\n' + 'Ignoring those options for {}'.format(name)) + repo_url = None + tag = '' + ref_hash = '' + branch = '' + + if repo_path is not None: + repo_url = repo_path + else: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + + if not from_submodule: + self._config.set(name, ExternalsDescription.REPO_URL, repo_url) self._config.set(name, ExternalsDescription.REQUIRED, str(required)) @@ -268,6 +309,12 @@ def create_section(self, repo_type, name, tag='', branch='', if externals: self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sparse: + self._config.set(name, ExternalsDescription.SPARSE, sparse) + + if from_submodule: + self._config.set(name, ExternalsDescription.SUBMODULE, "True") + def create_section_ext_only(self, name, required=True, externals=CFG_SUB_NAME): """Create a config section with autofilling some items and handling @@ -377,7 +424,7 @@ def update_branch(self, dest_dir, name, branch, repo_type=None, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): """Update a repository branch, and potentially the remote. @@ -391,7 +438,7 @@ def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_tag(self, dest_dir, name, tag, repo_type=None, filename=CFG_NAME, remove_branch=True): @@ -416,7 +463,7 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_underspecify_branch_tag(self, dest_dir, name, filename=CFG_NAME): @@ -435,7 +482,7 @@ def update_underspecify_branch_tag(self, dest_dir, name, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_underspecify_remove_url(self, dest_dir, name, filename=CFG_NAME): @@ -448,7 +495,7 @@ def update_underspecify_remove_url(self, dest_dir, name, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_protocol(self, dest_dir, name, protocol, repo_type=None, filename=CFG_NAME): @@ -461,7 +508,7 @@ def update_protocol(self, dest_dir, name, protocol, repo_type=None, repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) class BaseTestSysCheckout(unittest.TestCase): @@ -513,7 +560,7 @@ def tearDown(self): # return to our common starting point os.chdir(self._return_dir) - def setup_test_repo(self, parent_repo_name): + def setup_test_repo(self, parent_repo_name, dest_dir_in=None): """Setup the paths and clone the base test repo """ @@ -522,8 +569,12 @@ def setup_test_repo(self, parent_repo_name): print("Test repository name: {0}".format(test_dir_name)) parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - test_dir_name) + if dest_dir_in is None: + dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], + test_dir_name) + else: + dest_dir = dest_dir_in + # pylint: disable=W0212 GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) return dest_dir @@ -684,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): name = './{0}/mixed_req'.format(directory) self._check_generic_modified_ok_required(tree, name) + def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_empty_default_required(tree, name) + + def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_ok_clean_required(tree, name) + # ---------------------------------------------------------------- # # Check results for groups of externals under specific conditions @@ -844,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) + def _check_container_sparse_pre_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_empty(tree) + self._check_simple_sparse_empty(tree) + + def _check_container_sparse_post_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_ok(tree) + self._check_simple_sparse_ok(tree) + + def _check_file_exists(self, repo_dir, pathname): + "Check that exists in " + self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) + + def _check_file_absent(self, repo_dir, pathname): + "Check that does not exist in " + self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals @@ -1208,6 +1284,14 @@ def test_container_full(self): self.status_args) self._check_container_full_post_checkout(overall, tree) + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + # update the mixed-use repo to point to different branch self._generator.update_branch(under_test_dir, 'mixed_req', 'new-feature', MIXED_REPO_NAME) @@ -1288,6 +1372,40 @@ def test_mixed_simple(self): self.status_args) self._check_mixed_cont_simple_required_post_checkout(overall, tree) + def test_container_sparse(self): + """Verify that 'full' container with simple subrepo + can run a sparse checkout and generate the correct initial status. + + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_sparse(under_test_dir) + + # inital checkout + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.checkout_args) + self._check_container_sparse_pre_checkout(overall, tree) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_sparse_post_checkout(overall, tree) + + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + subrepo_path = os.path.join('externals', 'simp_sparse') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + class TestSysCheckoutSVN(BaseTestSysCheckout): """Run systems level tests of checkout_externals accessing svn repositories @@ -1434,6 +1552,237 @@ def test_container_simple_svn(self): self.verbose_args) self._check_container_simple_svn_post_checkout(overall, tree) +class TestSubrepoCheckout(BaseTestSysCheckout): + # Need to store information at setUp time for checking + # pylint: disable=too-many-instance-attributes + """Run tests to ensure proper handling of repos with submodules. + + By default, submodules in git repositories are checked out. A git + repository checked out as a submodule is treated as if it was + listed in an external with the same properties as in the source + .gitmodules file. + """ + + def setUp(self): + """Setup for all submodule checkout tests + Create a repo with two submodule repositories. + """ + + # Run the basic setup + super(TestSubrepoCheckout, self).setUp() + # create test repo + # We need to do this here (rather than have a static repo) because + # git submodules do not allow for variables in .gitmodules files + self._test_repo_name = 'test_repo_with_submodules' + self._bare_branch_name = 'subrepo_branch' + self._config_branch_name = 'subrepo_config_branch' + self._container_extern_name = 'externals_container.cfg' + self._my_test_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], + self._test_id) + self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) + self._checkout_dir = 'repo_with_submodules' + check_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + dest_dir_in=self._repo_dir) + self.assertTrue(self._repo_dir == check_dir) + # Add the submodules + cwd = os.getcwd() + fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_NAME) + simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO_NAME) + self._simple_ext_fork_name = SIMPLE_FORK_NAME.split('.')[0] + self._simple_ext_name = SIMPLE_REPO_NAME.split('.')[0] + os.chdir(self._repo_dir) + # Add a branch with a subrepo + cmd = ['git', 'branch', self._bare_branch_name, 'master'] + execute_subprocess(cmd) + cmd = ['git', 'checkout', self._bare_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'submodule', 'add', fork_repo_dir] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-am', "'Added simple-ext-fork as a submodule'"] + execute_subprocess(cmd) + # Save the fork repo hash for comparison + os.chdir(self._simple_ext_fork_name) + self._fork_hash_check = self.get_git_hash() + os.chdir(self._repo_dir) + # Now, create a branch to test from_sbmodule + cmd = ['git', 'branch', + self._config_branch_name, self._bare_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'checkout', self._config_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'submodule', 'add', simple_repo_dir] + execute_subprocess(cmd) + # Checkout feature2 + os.chdir(self._simple_ext_name) + cmd = ['git', 'branch', 'feature2', 'origin/feature2'] + execute_subprocess(cmd) + cmd = ['git', 'checkout', 'feature2'] + execute_subprocess(cmd) + # Save the fork repo hash for comparison + self._simple_hash_check = self.get_git_hash() + os.chdir(self._repo_dir) + self.create_externals_file(filename=self._container_extern_name, + dest_dir=self._repo_dir, from_submodule=True) + cmd = ['git', 'add', self._container_extern_name] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-am', "'Added simple-ext as a submodule'"] + execute_subprocess(cmd) + # Reset to master + cmd = ['git', 'checkout', 'master'] + execute_subprocess(cmd) + os.chdir(cwd) + + @staticmethod + def get_git_hash(revision="HEAD"): + """Return the hash for """ + cmd = ['git', 'rev-parse', revision] + git_out = execute_subprocess(cmd, output_to_caller=True) + return git_out.strip() + + def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, + branch_name=None, sub_externals=None, + from_submodule=False): + # pylint: disable=too-many-arguments + """Create a container externals file with only simple externals. + + """ + self._generator.create_config() + + if dest_dir is None: + dest_dir = self._my_test_dir + + if from_submodule: + self._generator.create_section(SIMPLE_FORK_NAME, + self._simple_ext_fork_name, + from_submodule=True) + self._generator.create_section(SIMPLE_REPO_NAME, + self._simple_ext_name, + branch='feature3', path='', + from_submodule=False) + else: + if branch_name is None: + branch_name = 'master' + + self._generator.create_section(self._test_repo_name, + self._checkout_dir, + branch=branch_name, + path=name, externals=sub_externals, + repo_path=self._repo_dir) + + self._generator.write_config(dest_dir, filename=filename) + + def idempotence_check(self, checkout_dir): + """Verify that calling checkout_externals and + checkout_externals --status does not cause errors""" + cwd = os.getcwd() + os.chdir(checkout_dir) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.status_args) + self.assertTrue(overall == 0) + os.chdir(cwd) + + def test_submodule_checkout_bare(self): + """Verify that a git repo with submodule is properly checked out + This test if for where there is no 'externals' keyword in the + parent repo. + Correct behavior is that the submodule is checked out using + normal git submodule behavior. + """ + simple_ext_fork_tag = "(tag1)" + simple_ext_fork_status = " " + self.create_externals_file(branch_name=self._bare_branch_name) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertTrue(os.path.exists(fork_file)) + os.chdir(checkout_dir) + submods = git_submodule_status(checkout_dir) + self.assertEqual(len(submods.keys()), 1) + self.assertTrue(self._simple_ext_fork_name in submods) + submod = submods[self._simple_ext_fork_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._fork_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], simple_ext_fork_status) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], simple_ext_fork_tag) + os.chdir(cwd) + self.idempotence_check(checkout_dir) + + def test_submodule_checkout_none(self): + """Verify that a git repo with submodule is properly checked out + This test is for when 'externals=None' is in parent repo's + externals cfg file. + Correct behavior is the submodle is not checked out. + """ + self.create_externals_file(branch_name=self._bare_branch_name, + sub_externals="none") + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertFalse(os.path.exists(fork_file)) + os.chdir(cwd) + self.idempotence_check(checkout_dir) + + def test_submodule_checkout_config(self): # pylint: disable=too-many-locals + """Verify that a git repo with submodule is properly checked out + This test if for when the 'from_submodule' keyword is used in the + parent repo. + Correct behavior is that the submodule is checked out using + normal git submodule behavior. + """ + tag_check = None # Not checked out as submodule + status_check = "-" # Not checked out as submodule + self.create_externals_file(branch_name=self._config_branch_name, + sub_externals=self._container_extern_name) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertTrue(os.path.exists(fork_file)) + os.chdir(checkout_dir) + # Check submodule status + submods = git_submodule_status(checkout_dir) + self.assertEqual(len(submods.keys()), 2) + self.assertTrue(self._simple_ext_fork_name in submods) + submod = submods[self._simple_ext_fork_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._fork_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], status_check) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], tag_check) + self.assertTrue(self._simple_ext_name in submods) + submod = submods[self._simple_ext_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._simple_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], status_check) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], tag_check) + # Check fork repo status + os.chdir(self._simple_ext_fork_name) + self.assertEqual(self.get_git_hash(), self._fork_hash_check) + os.chdir(checkout_dir) + os.chdir(self._simple_ext_name) + hash_check = self.get_git_hash('origin/feature3') + self.assertEqual(self.get_git_hash(), hash_check) + os.chdir(cwd) + self.idempotence_check(checkout_dir) class TestSysCheckoutErrors(BaseTestSysCheckout): """Run systems level tests of error conditions in checkout_externals diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 5de60e4f35..637f760ee5 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -316,11 +316,13 @@ def setUp(self): """Create config object used as basis for all tests """ self._config = config_parser() + self._gmconfig = config_parser() self.setup_config() def setup_config(self): """Boiler plate construction of xml string for componet 1 """ + # Create a standard externals config with a single external name = 'test' self._config.add_section(name) self._config.set(name, ExternalsDescription.PATH, 'externals') @@ -332,6 +334,14 @@ def setup_config(self): self._config.add_section(DESCRIPTION_SECTION) self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + # Create a .gitmodules test + name = 'submodule "gitmodules_test"' + self._gmconfig.add_section(name) + self._gmconfig.set(name, "path", 'externals/test') + self._gmconfig.set(name, "url", '/path/to/repo') + # NOTE(goldy, 2019-03) Should test other possible keywords such as + # fetchRecurseSubmodules, ignore, and shallow + def test_cfg_v1_ok(self): """Test that a correct cfg v1 object is created by create_externals_description @@ -356,7 +366,7 @@ def test_dict(self): rdata = {ExternalsDescription.PROTOCOL: 'git', ExternalsDescription.REPO_URL: '/path/to/repo', ExternalsDescription.TAG: 'tagv1', - } + } desc = { 'test': { diff --git a/test/test_unit_repository.py b/test/test_unit_repository.py index 2152503c2d..5b9c242fd3 100644 --- a/test/test_unit_repository.py +++ b/test/test_unit_repository.py @@ -36,7 +36,8 @@ def setUp(self): ExternalsDescription.REPO_URL: 'junk_root', ExternalsDescription.TAG: 'junk_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } def test_create_repo_git(self): """Verify that several possible names for the 'git' protocol @@ -95,7 +96,8 @@ def test_tag(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.TAG: tag, ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.tag(), tag) @@ -112,7 +114,8 @@ def test_branch(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.branch(), branch) @@ -125,11 +128,13 @@ def test_hash(self): protocol = 'test_protocol' url = 'test_url' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: EMPTY_STR, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.hash(), ref) @@ -146,11 +151,13 @@ def test_tag_branch(self): branch = 'test_branch' tag = 'test_tag' ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -165,11 +172,13 @@ def test_tag_branch_hash(self): branch = 'test_branch' tag = 'test_tag' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -184,11 +193,13 @@ def test_no_tag_no_branch(self): branch = EMPTY_STR tag = EMPTY_STR ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index b025fbd429..4a0a334bb1 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -547,7 +547,8 @@ def setUp(self): ExternalsDescription.TAG: 'very_useful_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } self._repo = GitRepository('test', self._rdata) def test_remote_git_proto(self):