From e557ee1746b9572c83dea81eada8136bc818cf4f Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 8 May 2020 10:00:55 -0600 Subject: [PATCH] Squashed 'manage_externals/' changes from f1e9e99..14973c1 fde04e4 Merge pull request #138 from billsacks/add_python38_tests 37e4c4a Do not update dictionary in-place in loop 7e8474b Remove testing on mac os 7f41c56 Fix pylint issue 3065b0d Add travis-ci tests with python3.7 and python3.8 34fbf55 Add support for git sparse checkout 6c6ef9f Fix pylint errors 6a659ad Added test for sparse checkout and updated documentation 1443243 Support for git sparsecheckout via read-tree. a48558d Merge pull request #119 from gold2718/submodules f72ffe7 Do not try git submodule update if no .gitmodules file (git bug) 804e0af Fix a pylint error 45aef95 Addressed review concerns 7da5031 New capability to use git submodule information to checkout externals 1926530 Merge pull request #118 from mnlevy1981/svn_switch b1b028d Updates after testing 9ea73e6 Add --svn-ignore-ancestry argument fc5acda Merge pull request #114 from billsacks/fix_large_output_hang aa2eb71 Try getting travis-ci working on MacOS 96842b4 Fix pylint errors 813fe3c pylint: disable useless-object-inheritance c49d878 Rework execute_subprocess timeout handling 8fc0e5f Cleanup from 'make style' b0b23a6 Merge pull request #110 from gold2718/help_fix 3cbcd16 Fixed and clarified help documentation 025e6cb Merge pull request #107 from jedwards4b/ignore_empty_git_dir 489842b if you encounter an empty directory clone into it 0c5a2f6 Merge pull request #106 from billsacks/remove_logfile_message 7799e99 Remove message about checking the log file for more details 0427305 Merge pull request #103 from billsacks/no_logging 9bb46aa Make no-logging be the default 9af6b02 Merge pull request #102 from billsacks/explain_qmark 7f973ae Run through make style d077a57 Add message describing meaning of '?' 60fc03b Merge pull request #101 from ESMCI/catch_svn_error 28073ec add exception class 4fb7e47 catch errors from svn status --xml bfa4831 Merge pull request #98 from billsacks/quieter 7d12650 make style afb4f11 Make more git and svn commands quieter a465b4f add --quiet argument to improve performance b2f3ae8 Merge pull request #83 from jedwards4b/jedwards/components_arg 3f4c88f fix comment c1b5b09 remove unneeded logic 4fdf180 one more test f78d60f another test bf52ac6 add a test 91d4851 fix pylint issue 987df5a only use components if populated 98a810d add a components arg to checkout only select components 6923119 Merge pull request #90 from ESMCI/issue-86-detached-sync-status b11ad61 Merge branch 'master' into issue-86-detached-sync-status 3b624cf Merge pull request #93 from billsacks/work_on_coverage 2562830 Run a single coverage command rather than two separate commands d1de5f8 Return to starting directory after each test 144f7d9 Merge pull request #92 from billsacks/point_to_esmci 58b8d3e Point to location of repository 0b46d81 Point to correct location for build/coverage status a385070 fix pylint problems dcf17b6 make style cleanup 92d342c Rewrite _current_ref to use plumbing rather than parsing porcelain ca0a5d3 Rework some git repository functions, and major rework of unit tests 719383e Remove commented-out pdb.set_trace() call 376c780 Bugfix: detect and report 'detached from' correctly 21813e9 Add system test demonstrating failure to detect out of sync status. 1a7c59d Merge documentation update into master. 247fee1 Document return values of checkout.py: main git-subtree-dir: manage_externals git-subtree-split: 14973c1cadf0436638d012d128766af27e9a1090 --- .travis.yml | 17 +- README.md | 25 +- manic/__init__.py | 2 +- manic/checkout.py | 134 +++- manic/externals_description.py | 373 ++++++++-- manic/repository.py | 20 +- manic/repository_factory.py | 4 +- manic/repository_git.py | 356 ++++++---- manic/repository_svn.py | 33 +- manic/sourcetree.py | 88 ++- manic/utils.py | 149 ++-- test/.pylint.rc | 2 +- test/Makefile | 13 +- .../14/2711fdbbcb8034d7cad6bae6801887b12fe61d | Bin 0 -> 83 bytes .../60/7ec299c17dd285c029edc41a0109e49d441380 | Bin 0 -> 168 bytes .../b7/692b6d391899680da7b9b6fd8af4c413f06fe7 | Bin 0 -> 137 bytes .../d1/163870d19c3dee34fada3a76b785cfa2a8424b | Bin 0 -> 130 bytes .../d8/ed2f33179d751937f8fde2e33921e4827babf4 | Bin 0 -> 60 bytes test/repos/simple-ext.git/refs/heads/master | 2 +- test/repos/simple-ext.git/refs/tags/tag2 | 1 + test/test_sys_checkout.py | 531 +++++++++++++- test/test_sys_repository_git.py | 238 +++++++ test/test_unit_externals_description.py | 12 +- test/test_unit_repository.py | 25 +- test/test_unit_repository_git.py | 653 +++++------------- 25 files changed, 1807 insertions(+), 871 deletions(-) create mode 100644 test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d create mode 100644 test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 create mode 100644 test/repos/simple-ext.git/objects/b7/692b6d391899680da7b9b6fd8af4c413f06fe7 create mode 100644 test/repos/simple-ext.git/objects/d1/163870d19c3dee34fada3a76b785cfa2a8424b create mode 100644 test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 create mode 100644 test/repos/simple-ext.git/refs/tags/tag2 create mode 100644 test/test_sys_repository_git.py diff --git a/.travis.yml b/.travis.yml index 5da83c56..1990cb96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,3 @@ -# NOTE(bja, 2017-11) travis-ci dosen't support python language builds -# on mac os. As a work around, we use built-in python on linux, and -# declare osx a 'generic' language, and create our own python env. - language: python os: linux python: @@ -9,16 +5,8 @@ python: - "3.4" - "3.5" - "3.6" -matrix: - include: - - os: osx - language: generic - before_install: - # NOTE(bja, 2017-11) update is slow, 2.7.12 installed by default, good enough! - # - brew update - # - brew outdated python2 || brew upgrade python2 - - virtualenv env -p python2 - - source env/bin/activate + - "3.7" + - "3.8" install: - pip install -r test/requirements.txt before_script: @@ -29,4 +17,3 @@ script: after_success: - cd test; make coverage - cd test; coveralls - \ No newline at end of file diff --git a/README.md b/README.md index 7750e6ef..c931c8e2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -- AUTOMATICALLY GENERATED FILE. DO NOT EDIT -- -[![Build Status](https://travis-ci.org/NCAR/manage_externals.svg?branch=master)](https://travis-ci.org/NCAR/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/NCAR/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/NCAR/manage_externals?branch=master) +[![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master) ``` usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace] [-d] [--no-logging] @@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. description file: $ cd ${SRC_ROOT} - $ ./manage_externals/checkout_externals --excernals my-externals.cfg + $ ./manage_externals/checkout_externals --externals my-externals.cfg * Status summary of the repositories managed by checkout_externals: @@ -202,4 +202,25 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below. Then the main 'externals' field in the top level repo should point to 'sub-externals.cfg'. + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + * Lines begining with '#' or ';' are comments and will be ignored. + +# Obtaining this tool, reporting issues, etc. + + The master repository for manage_externals is + https://github.com/ESMCI/manage_externals. Any issues with this tool + should be reported there. diff --git a/manic/__init__.py b/manic/__init__.py index e4d9b552..11badedd 100644 --- a/manic/__init__.py +++ b/manic/__init__.py @@ -1,7 +1,7 @@ """Public API for the manage_externals library """ -import manic.checkout as checkout +from manic import checkout from manic.utils import printlog __all__ = [ diff --git a/manic/checkout.py b/manic/checkout.py index 49834c55..edc56559 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -1,7 +1,7 @@ #!/usr/bin/env python """ -Tool to assemble respositories represented in a model-description file. +Tool to assemble repositories represented in a model-description file. If loaded as a module (e.g., in a component's buildcpp), it can be used to check the validity of existing subdirectories and load missing sources. @@ -20,7 +20,7 @@ from manic.externals_description import read_externals_description_file from manic.externals_status import check_safe_to_update_repos from manic.sourcetree import SourceTree -from manic.utils import printlog +from manic.utils import printlog, fatal_error from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME if sys.hexversion < 0x02070000: @@ -48,17 +48,9 @@ def commandline_arguments(args=None): description = ''' %(prog)s manages checking out groups of externals from revision -control based on a externals description file. By default only the +control based on an externals description file. By default only the required externals are checkout out. -Operations performed by manage_externals utilities are explicit and -data driven. %(prog)s will always make the working copy *exactly* -match what is in the externals file when modifying the working copy of -a repository. - -If %(prog)s isn't doing what you expected, double check the contents -of the externals description file. - Running %(prog)s without the '--status' option will always attempt to synchronize the working copy to exactly match the externals description. ''' @@ -76,7 +68,7 @@ def commandline_arguments(args=None): $ git clone git@github.com/{SOME_ORG}/some-project some-project-dev and you need to checkout the sub-project externals, then the root of the -source tree is /path/to/some-project-dev. Do *NOT* run %(prog)s +source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s from within /path/to/some-project-dev/sub-project The root of the source tree will be referred to as `${SRC_ROOT}` below. @@ -109,7 +101,7 @@ def commandline_arguments(args=None): description file: $ cd ${SRC_ROOT} - $ ./manage_externals/%(prog)s --excernals my-externals.cfg + $ ./manage_externals/%(prog)s --externals my-externals.cfg * Status summary of the repositories managed by %(prog)s: @@ -178,8 +170,9 @@ def commandline_arguments(args=None): Note: 'externals_only' will only process the external's own external description file without trying to manage a repository - for the component. This is used for retreiving externals for - standalone components like cam and clm. If the source root of the + for the component. This is used for retrieving externals for + standalone components like cam and ctsm which also serve as + sub-components within a larger project. If the source root of the externals_only component is the same as the main source root, then the local path must be set to '.', the unix current working directory, e. g. 'local_path = .' @@ -219,15 +212,54 @@ def commandline_arguments(args=None): * externals (string) : used to make manage_externals aware of sub-externals required by an external. This is a relative path to - the external's root directory. For example, the main externals - description has an external checkout out at 'src/useful_library'. - useful_library requires additional externals to be complete. - Those additional externals are managed from the source root by the - externals description file pointed 'useful_library/sub-xternals.cfg', - Then the main 'externals' field in the top level repo should point to - 'sub-externals.cfg'. + the external's root directory. For example, if LIBX is often used + as a sub-external, it might have an externals file (for its + externals) called Externals_LIBX.cfg. To use libx as a standalone + checkout, it would have another file, Externals.cfg with the + following entry: + + [ libx ] + local_path = . + protocol = externals_only + externals = Externals_LIBX.cfg + required = True + + Now, %(prog)s will process Externals.cfg and also process + Externals_LIBX.cfg as if it was a sub-external. + + * from_submodule (True / False) : used to pull the repo_url, local_path, + and hash properties for this external from the .gitmodules file in + this repository. Note that the section name (the entry in square + brackets) must match the name in the .gitmodules file. + If from_submodule is True, the protocol must be git and no repo_url, + local_path, hash, branch, or tag entries are allowed. + Default: False + + * sparse (string) : used to control a sparse checkout. This optional + entry should point to a filename (path relative to local_path) that + contains instructions on which repository paths to include (or + exclude) from the working tree. + See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree + Default: sparse checkout is disabled + + * Lines beginning with '#' or ';' are comments and will be ignored. + +# Obtaining this tool, reporting issues, etc. + + The master repository for manage_externals is + https://github.com/ESMCI/manage_externals. Any issues with this tool + should be reported there. + +# Troubleshooting - * Lines begining with '#' or ';' are comments and will be ignored. +Operations performed by manage_externals utilities are explicit and +data driven. %(prog)s will always attempt to make the working copy +*exactly* match what is in the externals file when modifying the +working copy of a repository. + +If %(prog)s is not doing what you expected, double check the contents +of the externals description file or examine the output of +./manage_externals/%(prog)s --status ''' @@ -238,6 +270,10 @@ def commandline_arguments(args=None): # # user options # + parser.add_argument("components", nargs="*", + help="Specific component(s) to checkout. By default, " + "all required externals are checked out.") + parser.add_argument('-e', '--externals', nargs='?', default='Externals.cfg', help='The externals description filename. ' @@ -249,9 +285,9 @@ def commandline_arguments(args=None): 'optional externals.') parser.add_argument('-S', '--status', action='store_true', default=False, - help='Output status of the repositories managed by ' + help='Output the status of the repositories managed by ' '%(prog)s. By default only summary information ' - 'is provided. Use verbose output to see details.') + 'is provided. Use the verbose option to see details.') parser.add_argument('-v', '--verbose', action='count', default=0, help='Output additional information to ' @@ -259,6 +295,13 @@ def commandline_arguments(args=None): 'used up to two times, increasing the ' 'verbosity level each time.') + parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, + help='By default, subversion will abort if a component is ' + 'already checked out and there is no common ancestry with ' + 'the new URL. This flag passes the "--ignore-ancestry" flag ' + 'to the svn switch call. (This is not recommended unless ' + 'you are sure about what you are doing.)') + # # developer options # @@ -270,8 +313,15 @@ def commandline_arguments(args=None): help='DEVELOPER: output additional debugging ' 'information to the screen and log file.') - parser.add_argument('--no-logging', action='store_true', - help='DEVELOPER: disable logging.') + logging_group = parser.add_mutually_exclusive_group() + + logging_group.add_argument('--logging', dest='do_logging', + action='store_true', + help='DEVELOPER: enable logging.') + logging_group.add_argument('--no-logging', dest='do_logging', + action='store_false', default=False, + help='DEVELOPER: disable logging ' + '(this is the default)') if args: options = parser.parse_args(args) @@ -290,8 +340,13 @@ def main(args): Function to call when module is called from the command line. Parse externals file and load required repositories or all repositories if the --all option is passed. + + Returns a tuple (overall_status, tree_status). overall_status is 0 + on success, non-zero on failure. tree_status gives the full status + *before* executing the checkout command - i.e., the status that it + used to determine if it's safe to proceed with the checkout. """ - if not args.no_logging: + if args.do_logging: logging.basicConfig(filename=LOG_FILE_NAME, format='%(levelname)s : %(asctime)s : %(message)s', datefmt='%Y-%m-%d %H:%M:%S', @@ -306,9 +361,16 @@ def main(args): root_dir = os.path.abspath(os.getcwd()) external_data = read_externals_description_file(root_dir, args.externals) - external = create_externals_description(external_data) + external = create_externals_description( + external_data, components=args.components) - source_tree = SourceTree(root_dir, external) + for comp in args.components: + if comp not in external.keys(): + fatal_error( + "No component {} found in {}".format( + comp, args.externals)) + + source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) printlog('Checking status of externals: ', end='') tree_status = source_tree.status() printlog('') @@ -338,13 +400,23 @@ def main(args): (2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually update out-of-sync externals (labeled with 's' above) as described in the configuration file {config_file}. + + +The external repositories labeled with '?' above are not under version +control using the expected protocol. If you are sure you want to switch +protocols, and you don't have any work you need to save from this +directory, then run "rm -rf [directory]" before re-running the +checkout_externals tool. """.format(program_name=program_name, config_file=args.externals) printlog('-' * 70) printlog(msg) printlog('-' * 70) else: - source_tree.checkout(args.verbose, load_all) + if not args.components: + source_tree.checkout(args.verbose, load_all) + for comp in args.components: + source_tree.checkout(args.verbose, load_all, load_comp=comp) printlog('') logging.info('%s completed without exceptions.', program_name) diff --git a/manic/externals_description.py b/manic/externals_description.py index 65569daa..b0c4f736 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -22,15 +22,17 @@ import os.path import re -# ConfigParser was renamed in python2 to configparser. In python2, -# ConfigParser returns byte strings, str, instead of unicode. We need -# unicode to be compatible with xml and json parser and python3. +# ConfigParser in python2 was renamed to configparser in python3. +# In python2, ConfigParser returns byte strings, str, instead of unicode. +# We need unicode to be compatible with xml and json parser and python3. try: # python2 from ConfigParser import SafeConfigParser as config_parser from ConfigParser import MissingSectionHeaderError from ConfigParser import NoSectionError, NoOptionError + USE_PYTHON2 = True + def config_string_cleaner(text): """convert strings into unicode """ @@ -41,6 +43,8 @@ def config_string_cleaner(text): from configparser import MissingSectionHeaderError from configparser import NoSectionError, NoOptionError + USE_PYTHON2 = False + def config_string_cleaner(text): """Python3 already uses unicode strings, so just return the string without modification. @@ -49,6 +53,7 @@ def config_string_cleaner(text): return text from .utils import printlog, fatal_error, str_to_bool, expand_local_url +from .utils import execute_subprocess from .global_constants import EMPTY_STR, PPRINTER, VERSION_SEPERATOR # @@ -59,8 +64,8 @@ def config_string_cleaner(text): def read_externals_description_file(root_dir, file_name): - """Given a file name containing a externals description, determine the - format and read it into it's internal representation. + """Read a file containing an externals description and + create its internal representation. """ root_dir = os.path.abspath(root_dir) @@ -70,37 +75,204 @@ def read_externals_description_file(root_dir, file_name): file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): - msg = ('ERROR: Model description file, "{0}", does not ' - 'exist at path:\n {1}\nDid you run from the root of ' - 'the source tree?'.format(file_name, file_path)) + if file_name.lower() == "none": + msg = ('INTERNAL ERROR: Attempt to read externals file ' + 'from {0} when not configured'.format(file_path)) + else: + msg = ('ERROR: Model description file, "{0}", does not ' + 'exist at path:\n {1}\nDid you run from the root of ' + 'the source tree?'.format(file_name, file_path)) + + fatal_error(msg) + + externals_description = None + if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: + externals_description = read_gitmodules_file(root_dir, file_name) + else: + try: + config = config_parser() + config.read(file_path) + externals_description = config + except MissingSectionHeaderError: + # not a cfg file + pass + + if externals_description is None: + msg = 'Unknown file format!' + fatal_error(msg) + + return externals_description + +class LstripReader(object): + "LstripReader formats .gitmodules files to be acceptable for configparser" + def __init__(self, filename): + with open(filename, 'r') as infile: + lines = infile.readlines() + self._lines = list() + self._num_lines = len(lines) + self._index = 0 + for line in lines: + self._lines.append(line.lstrip()) + + def readlines(self): + """Return all the lines from this object's file""" + return self._lines + + def readline(self, size=-1): + """Format and return the next line or raise StopIteration""" + try: + line = self.next() + except StopIteration: + line = '' + + if (size > 0) and (len(line) < size): + return line[0:size] + + return line + + def __iter__(self): + """Begin an iteration""" + self._index = 0 + return self + + def next(self): + """Return the next line or raise StopIteration""" + if self._index >= self._num_lines: + raise StopIteration + + self._index = self._index + 1 + return self._lines[self._index - 1] + + def __next__(self): + return self.next() + +def git_submodule_status(repo_dir): + """Run the git submodule status command to obtain submodule hashes. + """ + # This function is here instead of GitRepository to avoid a dependency loop + cwd = os.getcwd() + os.chdir(repo_dir) + cmd = ['git', 'submodule', 'status'] + git_output = execute_subprocess(cmd, output_to_caller=True) + submodules = {} + submods = git_output.split('\n') + for submod in submods: + if submod: + status = submod[0] + items = submod[1:].split(' ') + if len(items) > 2: + tag = items[2] + else: + tag = None + + submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} + + os.chdir(cwd) + return submodules + +def parse_submodules_desc_section(section_items, file_path): + """Find the path and url for this submodule description""" + path = None + url = None + for item in section_items: + name = item[0].strip().lower() + if name == 'path': + path = item[1].strip() + elif name == 'url': + url = item[1].strip() + else: + msg = 'WARNING: Ignoring unknown {} property, in {}' + msg = msg.format(item[0], file_path) # fool pylint + logging.warning(msg) + + return path, url + +def read_gitmodules_file(root_dir, file_name): + # pylint: disable=deprecated-method + # Disabling this check because the method is only used for python2 + """Read a .gitmodules file and convert it to be compatible with an + externals description. + """ + root_dir = os.path.abspath(root_dir) + msg = 'In directory : {0}'.format(root_dir) + logging.info(msg) + printlog('Processing submodules description file : {0}'.format(file_name)) + + file_path = os.path.join(root_dir, file_name) + if not os.path.exists(file_name): + msg = ('ERROR: submodules description file, "{0}", does not ' + 'exist at path:\n {1}'.format(file_name, file_path)) fatal_error(msg) + submodules_description = None externals_description = None try: config = config_parser() - config.read(file_path) - externals_description = config + if USE_PYTHON2: + config.readfp(LstripReader(file_path), filename=file_name) + else: + config.read_file(LstripReader(file_path), source=file_name) + + submodules_description = config except MissingSectionHeaderError: # not a cfg file pass - if externals_description is None: + if submodules_description is None: msg = 'Unknown file format!' fatal_error(msg) + else: + # Convert the submodules description to an externals description + externals_description = config_parser() + # We need to grab all the commit hashes for this repo + submods = git_submodule_status(root_dir) + for section in submodules_description.sections(): + if section[0:9] == 'submodule': + sec_name = section[9:].strip(' "') + externals_description.add_section(sec_name) + section_items = submodules_description.items(section) + path, url = parse_submodules_desc_section(section_items, + file_path) + + if path is None: + msg = 'Submodule {} missing path'.format(sec_name) + fatal_error(msg) - return externals_description + if url is None: + msg = 'Submodule {} missing url'.format(sec_name) + fatal_error(msg) + + externals_description.set(sec_name, + ExternalsDescription.PATH, path) + externals_description.set(sec_name, + ExternalsDescription.PROTOCOL, 'git') + externals_description.set(sec_name, + ExternalsDescription.REPO_URL, url) + externals_description.set(sec_name, + ExternalsDescription.REQUIRED, 'True') + git_hash = submods[sec_name]['hash'] + externals_description.set(sec_name, + ExternalsDescription.HASH, git_hash) + + # Required items + externals_description.add_section(DESCRIPTION_SECTION) + externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + return externals_description -def create_externals_description(model_data, model_format='cfg'): +def create_externals_description( + model_data, model_format='cfg', components=None, parent_repo=None): """Create the a externals description object from the provided data """ externals_description = None if model_format == 'dict': - externals_description = ExternalsDescriptionDict(model_data, ) + externals_description = ExternalsDescriptionDict( + model_data, components=components) elif model_format == 'cfg': major, _, _ = get_cfg_schema_version(model_data) if major == 1: - externals_description = ExternalsDescriptionConfigV1(model_data) + externals_description = ExternalsDescriptionConfigV1( + model_data, components=components, parent_repo=parent_repo) else: msg = ('Externals description file has unsupported schema ' 'version "{0}".'.format(major)) @@ -170,18 +342,21 @@ class ExternalsDescription(dict): # keywords defining the interface into the externals description data EXTERNALS = 'externals' BRANCH = 'branch' - REPO = 'repo' - REQUIRED = 'required' - TAG = 'tag' + SUBMODULE = 'from_submodule' + HASH = 'hash' + NAME = 'name' PATH = 'local_path' PROTOCOL = 'protocol' + REPO = 'repo' REPO_URL = 'repo_url' - HASH = 'hash' - NAME = 'name' + REQUIRED = 'required' + TAG = 'tag' + SPARSE = 'sparse' PROTOCOL_EXTERNALS_ONLY = 'externals_only' PROTOCOL_GIT = 'git' PROTOCOL_SVN = 'svn' + GIT_SUBMODULES_FILENAME = '.gitmodules' KNOWN_PRROTOCOLS = [PROTOCOL_GIT, PROTOCOL_SVN, PROTOCOL_EXTERNALS_ONLY] # v1 xml keywords @@ -194,15 +369,17 @@ class ExternalsDescription(dict): _source_schema = {REQUIRED: True, PATH: 'string', EXTERNALS: 'string', + SUBMODULE : True, REPO: {PROTOCOL: 'string', REPO_URL: 'string', TAG: 'string', BRANCH: 'string', HASH: 'string', - } - } + SPARSE: 'string', + } + } - def __init__(self): + def __init__(self, parent_repo=None): """Convert the xml into a standardized dict that can be used to construct the source objects @@ -215,6 +392,7 @@ def __init__(self): self._input_major = None self._input_minor = None self._input_patch = None + self._parent_repo = parent_repo def _verify_schema_version(self): """Use semantic versioning rules to verify we can process this schema. @@ -262,6 +440,7 @@ def _check_user_input(self): self._validate() def _check_data(self): + # pylint: disable=too-many-branches,too-many-statements """Check user supplied data is valid where possible. """ for ext_name in self.keys(): @@ -279,6 +458,13 @@ def _check_data(self): ext_name)) fatal_error(msg) + if ((self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_GIT) + and (self.SUBMODULE in self[ext_name])): + msg = ('self.SUBMODULE is only supported with {0} protocol, ' + '"{1}" is defined as an {2} repository') + fatal_error(msg.format(self.PROTOCOL_GIT, ext_name, + self[ext_name][self.REPO][self.PROTOCOL])) + if (self[ext_name][self.REPO][self.PROTOCOL] != self.PROTOCOL_EXTERNALS_ONLY): ref_count = 0 @@ -298,11 +484,23 @@ def _check_data(self): found_refs = '"{0} = {1}", {2}'.format( self.HASH, self[ext_name][self.REPO][self.HASH], found_refs) + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + ref_count += 1 + found_refs = '"{0} = {1}", {2}'.format( + self.SUBMODULE, + self[ext_name][self.SUBMODULE], found_refs) if ref_count > 1: - msg = ('Model description is over specified! Only one of ' - '"tag", "branch", or "hash" may be specified for ' - 'repo description of "{0}".'.format(ext_name)) + msg = 'Model description is over specified! ' + if self.SUBMODULE in self[ext_name]: + msg += ('from_submodule is not compatible with ' + '"tag", "branch", or "hash" ') + else: + msg += (' Only one of "tag", "branch", or "hash" ' + 'may be specified ') + + msg += 'for repo description of "{0}".'.format(ext_name) msg = '{0}\nFound: {1}'.format(msg, found_refs) fatal_error(msg) elif ref_count < 1: @@ -311,17 +509,39 @@ def _check_data(self): 'repo description of "{0}"'.format(ext_name)) fatal_error(msg) - if self.REPO_URL not in self[ext_name][self.REPO]: + if (self.REPO_URL not in self[ext_name][self.REPO] and + (self.SUBMODULE not in self[ext_name] or + not self[ext_name][self.SUBMODULE])): msg = ('Model description is under specified! Must have ' '"repo_url" in repo ' 'description for "{0}"'.format(ext_name)) fatal_error(msg) - url = expand_local_url( - self[ext_name][self.REPO][self.REPO_URL], ext_name) - self[ext_name][self.REPO][self.REPO_URL] = url + if (self.SUBMODULE in self[ext_name] and + self[ext_name][self.SUBMODULE]): + if self.REPO_URL in self[ext_name][self.REPO]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible ' + 'with {0} keyword for'.format(self.REPO_URL)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.PATH in self[ext_name]: + msg = ('Model description is over specified! ' + 'from_submodule keyword is not compatible with ' + '{0} keyword for'.format(self.PATH)) + msg = '{0} repo description of "{1}"'.format(msg, + ext_name) + fatal_error(msg) + + if self.REPO_URL in self[ext_name][self.REPO]: + url = expand_local_url( + self[ext_name][self.REPO][self.REPO_URL], ext_name) + self[ext_name][self.REPO][self.REPO_URL] = url def _check_optional(self): + # pylint: disable=too-many-branches """Some fields like externals, repo:tag repo:branch are (conditionally) optional. We don't want the user to be required to enter them in every externals description file, but @@ -329,6 +549,7 @@ def _check_optional(self): default values if appropriate. """ + submod_desc = None # Only load submodules info once for field in self: # truely optional if self.EXTERNALS not in self[field]: @@ -343,6 +564,72 @@ def _check_optional(self): self[field][self.REPO][self.HASH] = EMPTY_STR if self.REPO_URL not in self[field][self.REPO]: self[field][self.REPO][self.REPO_URL] = EMPTY_STR + if self.SPARSE not in self[field][self.REPO]: + self[field][self.REPO][self.SPARSE] = EMPTY_STR + + # from_submodule has a complex relationship with other fields + if self.SUBMODULE in self[field]: + # User wants to use submodule information, is it available? + if self._parent_repo is None: + # No parent == no submodule information + PPRINTER.pprint(self[field]) + msg = 'No parent submodule for "{0}"'.format(field) + fatal_error(msg) + elif self._parent_repo.protocol() != self.PROTOCOL_GIT: + PPRINTER.pprint(self[field]) + msg = 'Parent protocol, "{0}", does not support submodules' + fatal_error(msg.format(self._parent_repo.protocol())) + else: + args = self._repo_config_from_submodule(field, submod_desc) + repo_url, repo_path, ref_hash, submod_desc = args + + if repo_url is None: + msg = ('Cannot checkout "{0}" as a submodule, ' + 'repo not found in {1} file') + fatal_error(msg.format(field, + self.GIT_SUBMODULES_FILENAME)) + # Fill in submodule fields + self[field][self.REPO][self.REPO_URL] = repo_url + self[field][self.REPO][self.HASH] = ref_hash + self[field][self.PATH] = repo_path + + if self[field][self.SUBMODULE]: + # We should get everything from the parent submodule + # configuration. + pass + # No else (from _submodule = False is the default) + else: + # Add the default value (not using submodule information) + self[field][self.SUBMODULE] = False + + def _repo_config_from_submodule(self, field, submod_desc): + """Find the external config information for a repository from + its submodule configuration information. + """ + if submod_desc is None: + repo_path = os.getcwd() # Is this always correct? + submod_file = self._parent_repo.submodules_file(repo_path=repo_path) + if submod_file is None: + msg = ('Cannot checkout "{0}" from submodule information\n' + ' Parent repo, "{1}" does not have submodules') + fatal_error(msg.format(field, self._parent_repo.name())) + + submod_file = read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_file) + + # Can we find our external? + repo_url = None + repo_path = None + ref_hash = None + for ext_field in submod_desc: + if field == ext_field: + ext = submod_desc[ext_field] + repo_url = ext[self.REPO][self.REPO_URL] + repo_path = ext[self.PATH] + ref_hash = ext[self.REPO][self.HASH] + break + + return repo_url, repo_path, ref_hash, submod_desc def _validate(self): """Validate that the parsed externals description contains all necessary @@ -380,11 +667,12 @@ def validate_data_struct(schema, data): if isinstance(schema, dict) and isinstance(data, dict): # Both are dicts, recursively verify that all fields # in schema are present in the data. - for k in schema: - in_ref = in_ref and (k in data) + for key in schema: + in_ref = in_ref and (key in data) if in_ref: valid = valid and ( - validate_data_struct(schema[k], data[k])) + validate_data_struct(schema[key], data[key])) + is_valid = in_ref and valid else: # non-recursive structure. verify data and schema have @@ -419,7 +707,7 @@ class ExternalsDescriptionDict(ExternalsDescription): """ - def __init__(self, model_data): + def __init__(self, model_data, components=None): """Parse a native dictionary into a externals description. """ ExternalsDescription.__init__(self) @@ -430,6 +718,11 @@ def __init__(self, model_data): self._input_minor = 0 self._input_patch = 0 self._verify_schema_version() + if components: + for key in model_data.items(): + if key not in components: + del model_data[key] + self.update(model_data) self._check_user_input() @@ -440,12 +733,12 @@ class ExternalsDescriptionConfigV1(ExternalsDescription): """ - def __init__(self, model_data): - """Convert the xml into a standardized dict that can be used to + def __init__(self, model_data, components=None, parent_repo=None): + """Convert the config data into a standardized dict that can be used to construct the source objects """ - ExternalsDescription.__init__(self) + ExternalsDescription.__init__(self, parent_repo=parent_repo) self._schema_major = 1 self._schema_minor = 1 self._schema_patch = 0 @@ -453,7 +746,7 @@ def __init__(self, model_data): get_cfg_schema_version(model_data) self._verify_schema_version() self._remove_metadata(model_data) - self._parse_cfg(model_data) + self._parse_cfg(model_data, components=components) self._check_user_input() @staticmethod @@ -465,7 +758,7 @@ def _remove_metadata(model_data): """ model_data.remove_section(DESCRIPTION_SECTION) - def _parse_cfg(self, cfg_data): + def _parse_cfg(self, cfg_data, components=None): """Parse a config_parser object into a externals description. """ def list_to_dict(input_list, convert_to_lower_case=True): @@ -482,6 +775,8 @@ def list_to_dict(input_list, convert_to_lower_case=True): for section in cfg_data.sections(): name = config_string_cleaner(section.lower().strip()) + if components and name not in components: + continue self[name] = {} self[name].update(list_to_dict(cfg_data.items(section))) self[name][self.REPO] = {} diff --git a/manic/repository.py b/manic/repository.py index d01849d3..ea4230fb 100644 --- a/manic/repository.py +++ b/manic/repository.py @@ -21,6 +21,7 @@ def __init__(self, component_name, repo): self._branch = repo[ExternalsDescription.BRANCH] self._hash = repo[ExternalsDescription.HASH] self._url = repo[ExternalsDescription.REPO_URL] + self._sparse = repo[ExternalsDescription.SPARSE] if self._url is EMPTY_STR: fatal_error('repo must have a URL') @@ -40,12 +41,14 @@ def __init__(self, component_name, repo): fatal_error('repo {0} must have exactly one of ' 'tag, branch or hash.'.format(self._name)) - def checkout(self, base_dir_path, repo_dir_name, verbosity): # pylint: disable=unused-argument + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the source. If the repo destination directory does not exist, checkout the correce branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) """ msg = ('DEV_ERROR: checkout method must be implemented in all ' 'repository classes! {0}'.format(self.__class__.__name__)) @@ -59,6 +62,11 @@ def status(self, stat, repo_dir_path): # pylint: disable=unused-argument 'repository classes! {0}'.format(self.__class__.__name__)) fatal_error(msg) + def submodules_file(self, repo_path=None): + # pylint: disable=no-self-use,unused-argument + """Stub for use by non-git VC systems""" + return None + def url(self): """Public access of repo url. """ @@ -78,3 +86,13 @@ def hash(self): """Public access of repo hash. """ return self._hash + + def name(self): + """Public access of repo name. + """ + return self._name + + def protocol(self): + """Public access of repo protocol. + """ + return self._protocol diff --git a/manic/repository_factory.py b/manic/repository_factory.py index c95e7a50..80a92a9d 100644 --- a/manic/repository_factory.py +++ b/manic/repository_factory.py @@ -11,7 +11,7 @@ from .utils import fatal_error -def create_repository(component_name, repo_info): +def create_repository(component_name, repo_info, svn_ignore_ancestry=False): """Determine what type of repository we have, i.e. git or svn, and create the appropriate object. @@ -20,7 +20,7 @@ def create_repository(component_name, repo_info): if protocol == 'git': repo = GitRepository(component_name, repo_info) elif protocol == 'svn': - repo = SvnRepository(component_name, repo_info) + repo = SvnRepository(component_name, repo_info, ignore_ancestry=svn_ignore_ancestry) elif protocol == 'externals_only': repo = None else: diff --git a/manic/repository_git.py b/manic/repository_git.py index f9e88df7..f9860510 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -7,12 +7,12 @@ import copy import os -import re from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR from .global_constants import VERBOSITY_VERBOSE from .repository import Repository from .externals_status import ExternalStatus +from .externals_description import ExternalsDescription, git_submodule_status from .utils import expand_local_url, split_remote_url, is_remote_url from .utils import fatal_error, printlog from .utils import execute_subprocess @@ -37,37 +37,40 @@ class GitRepository(Repository): """ - # match XYZ of '* (HEAD detached at {XYZ}): - # e.g. * (HEAD detached at origin/feature-2) - RE_DETACHED = re.compile( - r'\* \((?:[\w]+[\s]+)?detached (?:at|from) ([\w\-./]+)\)') - - # match tracking reference info, return XYZ from [XYZ] - # e.g. [origin/master] - RE_TRACKING = re.compile(r'\[([\w\-./]+)(?::[\s]+[\w\s,]+)?\]') - def __init__(self, component_name, repo): """ Parse repo (a XML element). """ Repository.__init__(self, component_name, repo) + self._gitmodules = None + self._submods = None # ---------------------------------------------------------------- # # Public API, defined by Repository # # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the source. - If the repo destination directory does not exist, checkout the correce + If the repo destination directory does not exist, checkout the correct branch or tag. """ repo_dir_path = os.path.join(base_dir_path, repo_dir_name) - if not os.path.exists(repo_dir_path): + repo_dir_exists = os.path.exists(repo_dir_path) + if (repo_dir_exists and not os.listdir( + repo_dir_path)) or not repo_dir_exists: self._clone_repo(base_dir_path, repo_dir_name, verbosity) - self._checkout_ref(repo_dir_path, verbosity) + self._checkout_ref(repo_dir_path, verbosity, recursive) + gmpath = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_dir_path) + else: + self._gitmodules = None + self._submods = None def status(self, stat, repo_dir_path): """ @@ -80,6 +83,16 @@ def status(self, stat, repo_dir_path): if os.path.exists(repo_dir_path): self._status_summary(stat, repo_dir_path) + def submodules_file(self, repo_path=None): + if repo_path is not None: + gmpath = os.path.join(repo_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + if os.path.exists(gmpath): + self._gitmodules = gmpath + self._submods = git_submodule_status(repo_path) + + return self._gitmodules + # ---------------------------------------------------------------- # # Internal work functions @@ -93,81 +106,42 @@ def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): self._git_clone(self._url, repo_dir_name, verbosity) os.chdir(cwd) - def _current_ref_from_branch_command(self, git_output): - """Parse output of the 'git branch -vv' command to determine the current - branch. The line starting with '*' is the current branch. It - can be one of the following head states: - - 1. On local branch - - feature2 36418b4 [origin/feature2] Work on feature2 - * feature3 36418b4 Work on feature2 - master 9b75494 [origin/master] Initialize repository. - - 2. Detached from sha - - * (HEAD detached at 36418b4) 36418b4 Work on feature2 - feature2 36418b4 [origin/feature2] Work on feature2 - master 9b75494 [origin/master] Initialize repository. - - 3. Detached from remote branch - - * (HEAD detached at origin/feature2) 36418b4 Work on feature2 - feature2 36418b4 [origin/feature2] Work on feature2 - feature3 36418b4 Work on feature2 - master 9b75494 [origin/master] Initialize repository. - - 4. Detached from tag - - * (HEAD detached at clm4_5_18_r272) b837fc36 clm4_5_18_r272 - - 5. On tracking branch. Note, may be may be ahead or behind remote. - - * master 562bac9a [origin/master] more test junk - - * master 408a8920 [origin/master: ahead 3] more junk - - * master 408a8920 [origin/master: ahead 3, behind 2] more junk + def _current_ref(self): + """Determine the *name* associated with HEAD. + + If we're on a branch, then returns the branch name; otherwise, + if we're on a tag, then returns the tag name; otherwise, returns + the current hash. Returns an empty string if no reference can be + determined (e.g., if we're not actually in a git repository). + """ + ref_found = False + + # If we're on a branch, then use that as the current ref + branch_found, branch_name = self._git_current_branch() + if branch_found: + current_ref = branch_name + ref_found = True + + if not ref_found: + # Otherwise, if we're exactly at a tag, use that as the + # current ref + tag_found, tag_name = self._git_current_tag() + if tag_found: + current_ref = tag_name + ref_found = True + + if not ref_found: + # Otherwise, use current hash as the current ref + hash_found, hash_name = self._git_current_hash() + if hash_found: + current_ref = hash_name + ref_found = True + + if not ref_found: + # If we still can't find a ref, return empty string. This + # can happen if we're not actually in a git repo + current_ref = '' - * master 822d687d [origin/master: behind 3] more junk - - NOTE: Parsing the output of the porcelain is probably not a - great idea, but there doesn't appear to be a single plumbing - command that will return the same info. - - """ - lines = git_output.splitlines() - ref = '' - for line in lines: - if line.startswith('*'): - ref = line - break - current_ref = EMPTY_STR - if not ref: - # not a git repo? some other error? we return so the - # caller can handle. - pass - elif 'detached' in ref: - match = self.RE_DETACHED.search(ref) - try: - current_ref = match.group(1) - except BaseException: - msg = 'DEV_ERROR: regex to detect detached head state failed!' - msg += '\nref:\n{0}\ngit_output\n{1}\n'.format(ref, git_output) - fatal_error(msg) - elif '[' in ref: - match = self.RE_TRACKING.search(ref) - try: - current_ref = match.group(1) - except BaseException: - msg = 'DEV_ERROR: regex to detect tracking branch failed.' - msg += '\nref:\n{0}\ngit_output\n{1}\n'.format(ref, git_output) - fatal_error(msg) - else: - # assumed local branch - current_ref = ref.split()[1] - - current_ref = current_ref.strip() return current_ref def _check_sync(self, stat, repo_dir_path): @@ -194,12 +168,11 @@ def _check_sync(self, stat, repo_dir_path): self._check_sync_logic(stat, repo_dir_path) def _check_sync_logic(self, stat, repo_dir_path): - """Isolate the complicated synce logic so it is not so deeply nested - and a bit easier to understand. - - Sync logic - only reporting on whether we are on the ref - (branch, tag, hash) specified in the externals description. + """Compare the underlying hashes of the currently checkout ref and the + expected ref. + Output: sets the sync_state as well as the current and + expected ref in the input status object. """ def compare_refs(current_ref, expected_ref): @@ -215,8 +188,8 @@ def compare_refs(current_ref, expected_ref): cwd = os.getcwd() os.chdir(repo_dir_path) - git_output = self._git_branch_vv() - current_ref = self._current_ref_from_branch_command(git_output) + # get the full hash of the current commit + _, current_ref = self._git_current_hash() if self._branch: if self._url == LOCAL_PATH_INDICATOR: @@ -230,22 +203,33 @@ def compare_refs(current_ref, expected_ref): else: expected_ref = "{0}/{1}".format(remote_name, self._branch) elif self._hash: - # NOTE(bja, 2018-03) For comparison purposes, we could - # determine which is longer and check that the short ref - # is a substring of the long ref. But it is simpler to - # just expand both to the full sha and do an exact - # comparison. - _, expected_ref = self._git_revparse_commit(self._hash) - _, current_ref = self._git_revparse_commit(current_ref) - else: + expected_ref = self._hash + elif self._tag: expected_ref = self._tag + else: + msg = 'In repo "{0}": none of branch, hash or tag are set'.format( + self._name) + fatal_error(msg) + + # record the *names* of the current and expected branches + stat.current_version = self._current_ref() + stat.expected_version = copy.deepcopy(expected_ref) - stat.sync_state = compare_refs(current_ref, expected_ref) if current_ref == EMPTY_STR: stat.sync_state = ExternalStatus.UNKNOWN - - stat.current_version = current_ref - stat.expected_version = expected_ref + else: + # get the underlying hash of the expected ref + revparse_status, expected_ref_hash = self._git_revparse_commit( + expected_ref) + if revparse_status: + # We failed to get the hash associated with + # expected_ref. Maybe we should assign this to some special + # status, but for now we're just calling this out-of-sync to + # remain consistent with how this worked before. + stat.sync_state = ExternalStatus.MODEL_MODIFIED + else: + # compare the underlying hashes + stat.sync_state = compare_refs(current_ref, expected_ref_hash) os.chdir(cwd) @@ -319,23 +303,30 @@ def _create_remote_name(self): remote_name = "{0}_{1}".format(base_name, repo_name) return remote_name - def _checkout_ref(self, repo_dir, verbosity): + def _checkout_ref(self, repo_dir, verbosity, submodules): """Checkout the user supplied reference + if is True, recursively initialize and update + the repo's submodules """ # import pdb; pdb.set_trace() cwd = os.getcwd() os.chdir(repo_dir) if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity) + self._checkout_local_ref(verbosity, submodules) else: - self._checkout_external_ref(verbosity) + self._checkout_external_ref(verbosity, submodules) + + if self._sparse: + self._sparse_checkout(repo_dir, verbosity) os.chdir(cwd) - def _checkout_local_ref(self, verbosity): + + def _checkout_local_ref(self, verbosity, submodules): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when checkout out the ref. - + if is True, recursively initialize and update + the repo's submodules """ if self._tag: ref = self._tag @@ -345,10 +336,12 @@ def _checkout_local_ref(self, verbosity): ref = self._hash self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity) + self._git_checkout_ref(ref, verbosity, submodules) - def _checkout_external_ref(self, verbosity): + def _checkout_external_ref(self, verbosity, submodules): """Checkout the reference from a remote repository + if is True, recursively initialize and update + the repo's submodules """ if self._tag: ref = self._tag @@ -363,14 +356,28 @@ def _checkout_external_ref(self, verbosity): self._git_remote_add(remote_name, self._url) self._git_fetch(remote_name) - # NOTE(bja, 2018-03) we need to send seperate ref and remote + # NOTE(bja, 2018-03) we need to send separate ref and remote # name to check_for_vaild_ref, but the combined name to # checkout_ref! self._check_for_valid_ref(ref, remote_name) if self._branch: ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity) + self._git_checkout_ref(ref, verbosity, submodules) + + def _sparse_checkout(self, repo_dir, verbosity): + """Use git read-tree to thin the working tree.""" + cwd = os.getcwd() + + cmd = ['cp', self._sparse, os.path.join(repo_dir, + '.git/info/sparse-checkout')] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + os.chdir(repo_dir) + self._git_sparse_checkout(verbosity) + + os.chdir(cwd) def _check_for_valid_ref(self, ref, remote_name=None): """Try some basic sanity checks on the user supplied reference so we @@ -595,14 +602,58 @@ def _status_v1z_is_dirty(git_output): # # ---------------------------------------------------------------- @staticmethod - def _git_branch_vv(): - """Run git branch -vv to obtain verbose branch information, including - upstream tracking and hash. + def _git_current_hash(): + """Return the full hash of the currently checked-out version. + Returns a tuple, (hash_found, hash), where hash_found is a + logical specifying whether a hash was found for HEAD (False + could mean we're not in a git repository at all). (If hash_found + is False, then hash is ''.) """ - cmd = ['git', 'branch', '--verbose', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output + status, git_output = GitRepository._git_revparse_commit("HEAD") + hash_found = not status + if not hash_found: + git_output = '' + return hash_found, git_output + + @staticmethod + def _git_current_branch(): + """Determines the name of the current branch. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a logical specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''.) + """ + cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = not status + if branch_found: + git_output = git_output.strip() + else: + git_output = '' + return branch_found, git_output + + @staticmethod + def _git_current_tag(): + """Determines the name tag corresponding to HEAD (if any). + + Returns a tuple, (tag_found, tag_name), where tag_found is a + logical specifying whether we found a tag name corresponding to + HEAD. (If tag_found is False, then tag_name is ''.) + """ + # git describe --exact-match --tags HEAD + cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + tag_found = not status + if tag_found: + git_output = git_output.strip() + else: + git_output = '' + return tag_found, git_output @staticmethod def _git_showref_tag(ref): @@ -647,6 +698,7 @@ def _git_revparse_commit(ref): '{0}^{1}'.format(ref, '{commit}'), ] status, git_output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True) + git_output = git_output.strip() return status, git_output @staticmethod @@ -679,6 +731,19 @@ def _git_remote_verbose(): git_output = execute_subprocess(cmd, output_to_caller=True) return git_output + @staticmethod + def has_submodules(repo_dir_path=None): + """Return True iff the repository at (or the current + directory if is None) has a '.gitmodules' file + """ + if repo_dir_path is None: + fname = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) + + return os.path.exists(fname) + # ---------------------------------------------------------------- # # system call to git for sideffects modifying the working tree @@ -688,34 +753,67 @@ def _git_remote_verbose(): def _git_clone(url, repo_dir_name, verbosity): """Run git clone for the side effect of creating a repository. """ - cmd = ['git', 'clone', url, repo_dir_name] + cmd = ['git', 'clone', '--quiet'] + subcmd = None + + cmd.extend([url, repo_dir_name]) if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) + if subcmd is not None: + os.chdir(repo_dir_name) + execute_subprocess(subcmd) @staticmethod def _git_remote_add(name, url): - """Run the git remote command to for the side effect of adding a remote + """Run the git remote command for the side effect of adding a remote """ cmd = ['git', 'remote', 'add', name, url] execute_subprocess(cmd) @staticmethod def _git_fetch(remote_name): - """Run the git fetch command to for the side effect of updating the repo + """Run the git fetch command for the side effect of updating the repo """ - cmd = ['git', 'fetch', '--tags', remote_name] + cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] execute_subprocess(cmd) @staticmethod - def _git_checkout_ref(ref, verbosity): - """Run the git checkout command to for the side effect of updating the repo + def _git_checkout_ref(ref, verbosity, submodules): + """Run the git checkout command for the side effect of updating the repo Param: ref is a reference to a local or remote object in the form 'origin/my_feature', or 'tag1'. """ - cmd = ['git', 'checkout', ref] + cmd = ['git', 'checkout', '--quiet', ref] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + if submodules: + GitRepository._git_update_submodules(verbosity) + + @staticmethod + def _git_sparse_checkout(verbosity): + """Configure repo via read-tree.""" + cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + execute_subprocess(cmd) + cmd = ['git', 'read-tree', '-mu', 'HEAD'] if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) + + @staticmethod + def _git_update_submodules(verbosity): + """Run git submodule update for the side effect of updating this + repo's submodules. + """ + # First, verify that we have a .gitmodules file + if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): + cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if verbosity >= VERBOSITY_VERBOSE: + printlog(' {0}'.format(' '.join(cmd))) + + execute_subprocess(cmd) diff --git a/manic/repository_svn.py b/manic/repository_svn.py index b11d36e6..408ed846 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -37,11 +37,12 @@ class SvnRepository(Repository): """ RE_URLLINE = re.compile(r'^URL:') - def __init__(self, component_name, repo): + def __init__(self, component_name, repo, ignore_ancestry=False): """ Parse repo (a XML element). """ Repository.__init__(self, component_name, repo) + self._ignore_ancestry = ignore_ancestry if self._branch: self._url = os.path.join(self._url, self._branch) elif self._tag: @@ -55,7 +56,7 @@ def __init__(self, component_name, repo): # Public API, defined by Repository # # ---------------------------------------------------------------- - def checkout(self, base_dir_path, repo_dir_name, verbosity): + def checkout(self, base_dir_path, repo_dir_name, verbosity, recursive): # pylint: disable=unused-argument """Checkout or update the working copy If the repo destination directory exists, switch the sandbox to @@ -63,13 +64,15 @@ def checkout(self, base_dir_path, repo_dir_name, verbosity): If the repo destination directory does not exist, checkout the correct branch or tag. + NB: is include as an argument for compatibility with + git functionality (repository_git.py) """ repo_dir_path = os.path.join(base_dir_path, repo_dir_name) if os.path.exists(repo_dir_path): cwd = os.getcwd() os.chdir(repo_dir_path) - self._svn_switch(self._url, verbosity) + self._svn_switch(self._url, self._ignore_ancestry, verbosity) # svn switch can lead to a conflict state, but it gives a # return code of 0. So now we need to make sure that we're # in a clean (non-conflict) state. @@ -137,9 +140,7 @@ def _abort_if_dirty(self, repo_dir_path, message): To recover: Clean up the above directory (resolving conflicts, etc.), then rerun checkout_externals. -""".format(cwd=repo_dir_path, - message=message, - status=status) +""".format(cwd=repo_dir_path, message=message, status=status) fatal_error(errmsg) @@ -205,7 +206,11 @@ def xml_status_is_dirty(svn_output): # pylint: enable=invalid-name is_dirty = False - xml_status = ET.fromstring(svn_output) + try: + xml_status = ET.fromstring(svn_output) + except BaseException: + fatal_error( + "SVN returned invalid XML message {}".format(svn_output)) xml_target = xml_status.find('./target') entries = xml_target.findall('./entry') for entry in entries: @@ -215,9 +220,8 @@ def xml_status_is_dirty(svn_output): continue if item == SVN_UNVERSIONED: continue - else: - is_dirty = True - break + is_dirty = True + break return is_dirty # ---------------------------------------------------------------- @@ -260,17 +264,20 @@ def _svn_checkout(url, repo_dir_path, verbosity): """ Checkout a subversion repository (repo_url) to checkout_dir. """ - cmd = ['svn', 'checkout', url, repo_dir_path] + cmd = ['svn', 'checkout', '--quiet', url, repo_dir_path] if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) @staticmethod - def _svn_switch(url, verbosity): + def _svn_switch(url, ignore_ancestry, verbosity): """ Switch branches for in an svn sandbox """ - cmd = ['svn', 'switch', url] + cmd = ['svn', 'switch', '--quiet'] + if ignore_ancestry: + cmd.append('--ignore-ancestry') + cmd.append(url) if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index a97851e6..b9c9c210 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -11,20 +11,20 @@ from .externals_description import read_externals_description_file from .externals_description import create_externals_description from .repository_factory import create_repository +from .repository_git import GitRepository from .externals_status import ExternalStatus from .utils import fatal_error, printlog from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR from .global_constants import VERBOSITY_VERBOSE - class _External(object): """ - _External represents an external object in side a SourceTree + _External represents an external object inside a SourceTree """ # pylint: disable=R0902 - def __init__(self, root_dir, name, ext_description): + def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): """Parse an external description file into a dictionary of externals. Input: @@ -37,12 +37,15 @@ def __init__(self, root_dir, name, ext_description): ext_description : dict - source ExternalsDescription object + svn_ignore_ancestry : bool - use --ignore-externals with svn switch + """ self._name = name self._repo = None self._externals = EMPTY_STR self._externals_sourcetree = None self._stat = ExternalStatus() + self._sparse = None # Parse the sub-elements # _path : local path relative to the containing source tree @@ -59,13 +62,20 @@ def __init__(self, root_dir, name, ext_description): self._required = ext_description[ExternalsDescription.REQUIRED] self._externals = ext_description[ExternalsDescription.EXTERNALS] - if self._externals: - self._create_externals_sourcetree() + # Treat a .gitmodules file as a backup externals config + if not self._externals: + if GitRepository.has_submodules(self._repo_dir_path): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + repo = create_repository( - name, ext_description[ExternalsDescription.REPO]) + name, ext_description[ExternalsDescription.REPO], + svn_ignore_ancestry=svn_ignore_ancestry) if repo: self._repo = repo + if self._externals and (self._externals.lower() != 'none'): + self._create_externals_sourcetree() + def get_name(self): """ Return the external object's name @@ -122,7 +132,7 @@ def status(self): if self._externals and self._externals_sourcetree: # we expect externals and they exist cwd = os.getcwd() - # SourceTree expecteds to be called from the correct + # SourceTree expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) ext_stats = self._externals_sourcetree.status(self._local_path) @@ -145,7 +155,7 @@ def checkout(self, verbosity, load_all): """ If the repo destination directory exists, ensure it is correct (from correct URL, correct branch or tag), and possibly update the external. - If the repo destination directory does not exist, checkout the correce + If the repo destination directory does not exist, checkout the correct branch or tag. If load_all is True, also load all of the the externals sub-externals. """ @@ -180,13 +190,14 @@ def checkout(self, verbosity, load_all): checkout_verbosity = verbosity - 1 else: checkout_verbosity = verbosity - self._repo.checkout(self._base_dir_path, - self._repo_dir_name, checkout_verbosity) + + self._repo.checkout(self._base_dir_path, self._repo_dir_name, + checkout_verbosity, self.clone_recursive()) def checkout_externals(self, verbosity, load_all): """Checkout the sub-externals for this object """ - if self._externals: + if self.load_externals(): if self._externals_sourcetree: # NOTE(bja, 2018-02): the subtree externals objects # were created during initial status check. Updating @@ -198,6 +209,24 @@ def checkout_externals(self, verbosity, load_all): self._create_externals_sourcetree() self._externals_sourcetree.checkout(verbosity, load_all) + def load_externals(self): + 'Return True iff an externals file should be loaded' + load_ex = False + if os.path.exists(self._repo_dir_path): + if self._externals: + if self._externals.lower() != 'none': + load_ex = os.path.exists(os.path.join(self._repo_dir_path, + self._externals)) + + return load_ex + + def clone_recursive(self): + 'Return True iff any .gitmodules files should be processed' + # Try recursive unless there is an externals entry + recursive = not self._externals + + return recursive + def _create_externals_sourcetree(self): """ """ @@ -210,6 +239,15 @@ def _create_externals_sourcetree(self): cwd = os.getcwd() os.chdir(self._repo_dir_path) + if self._externals.lower() == 'none': + msg = ('Internal: Attempt to create source tree for ' + 'externals = none in {}'.format(self._repo_dir_path)) + fatal_error(msg) + + if not os.path.exists(self._externals): + if GitRepository.has_submodules(): + self._externals = ExternalsDescription.GIT_SUBMODULES_FILENAME + if not os.path.exists(self._externals): # NOTE(bja, 2017-10) this check is redundent with the one # in read_externals_description_file! @@ -221,17 +259,17 @@ def _create_externals_sourcetree(self): externals_root = self._repo_dir_path model_data = read_externals_description_file(externals_root, self._externals) - externals = create_externals_description(model_data) + externals = create_externals_description(model_data, + parent_repo=self._repo) self._externals_sourcetree = SourceTree(externals_root, externals) os.chdir(cwd) - class SourceTree(object): """ SourceTree represents a group of managed externals """ - def __init__(self, root_dir, model): + def __init__(self, root_dir, model, svn_ignore_ancestry=False): """ Build a SourceTree object from a model description """ @@ -239,7 +277,7 @@ def __init__(self, root_dir, model): self._all_components = {} self._required_compnames = [] for comp in model: - src = _External(self._root_dir, comp, model[comp]) + src = _External(self._root_dir, comp, model[comp], svn_ignore_ancestry) self._all_components[comp] = src if model[comp][ExternalsDescription.REQUIRED]: self._required_compnames.append(comp) @@ -261,18 +299,20 @@ def status(self, relative_path_base=LOCAL_PATH_INDICATOR): for comp in load_comps: printlog('{0}, '.format(comp), end='') stat = self._all_components[comp].status() + stat_final = {} for name in stat.keys(): # check if we need to append the relative_path_base to # the path so it will be sorted in the correct order. - if not stat[name].path.startswith(relative_path_base): - stat[name].path = os.path.join(relative_path_base, - stat[name].path) - # store under key = updated path, and delete the - # old key. - comp_stat = stat[name] - del stat[name] - stat[comp_stat.path] = comp_stat - summary.update(stat) + if stat[name].path.startswith(relative_path_base): + # use as is, without any changes to path + stat_final[name] = stat[name] + else: + # append relative_path_base to path and store under key = updated path + modified_path = os.path.join(relative_path_base, + stat[name].path) + stat_final[modified_path] = stat[name] + stat_final[modified_path].path = modified_path + summary.update(stat_final) return summary diff --git a/manic/utils.py b/manic/utils.py index 824f5b9e..f57f4393 100644 --- a/manic/utils.py +++ b/manic/utils.py @@ -12,9 +12,9 @@ import os import subprocess import sys -import time +from threading import Timer -from .global_constants import LOCAL_PATH_INDICATOR, LOG_FILE_NAME +from .global_constants import LOCAL_PATH_INDICATOR # --------------------------------------------------------------------- # @@ -65,13 +65,15 @@ def last_n_lines(the_string, n_lines, truncation_message=None): lines = the_string.splitlines(True) if len(lines) <= n_lines: - return the_string + return_val = the_string else: lines_subset = lines[-n_lines:] str_truncated = ''.join(lines_subset) if truncation_message: str_truncated = truncation_message + '\n' + str_truncated - return str_truncated + return_val = str_truncated + + return return_val def indent_string(the_string, indent_level): @@ -119,9 +121,9 @@ def str_to_bool(bool_str): """ value = None str_lower = bool_str.lower() - if (str_lower == 'true') or (str_lower == 't'): + if str_lower in ('true', 't'): value = True - elif (str_lower == 'false') or (str_lower == 'f'): + elif str_lower in ('false', 'f'): value = False if value is None: msg = ('ERROR: invalid boolean string value "{0}". ' @@ -199,80 +201,30 @@ def expand_local_url(url, field): # subprocess # # --------------------------------------------------------------------- -_TIMEOUT_MSG = """ Timout errors typically occur when svn or git requires -authentication to access a private repository. On some systems, svn -and git requests for authentication information will not be displayed -to the user. In this case, the program will appear to hang and -generate a timeout error. Ensure you can run svn and git manually and -access all repositories without entering your authentication -information.""" - -_TIMEOUT_SEC = 300 -_POLL_DELTA_SEC = 0.02 - - -def _poll_subprocess(commands, status_to_caller, output_to_caller, - timeout_sec=_TIMEOUT_SEC): - """Create a subprocess and poll the process until complete. - Impose a timeout limit and checkout process output for known - conditions that require user interaction. +# Give the user a helpful message if we detect that a command seems to +# be hanging. +_HANGING_SEC = 300 - NOTE: the timeout_delta has significant impact on run time. If it - is too long, and the many quick local subprocess calls will - drastically increase the run time, especially in tests. - - NOTE: This function is broken out into for ease of - understanding. It does no error checking. It should only be called - from execute_subprocess, never directly. - - """ - logging.info(' '.join(commands)) - output = [] - start = time.time() - - proc = subprocess.Popen(commands, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - while proc.poll() is None: - time.sleep(_POLL_DELTA_SEC) - if time.time() - start > timeout_sec: - proc.kill() - time.sleep(_POLL_DELTA_SEC * 5) - msg = ("subprocess call to '{0}' has exceeded timeout limit of " - "{1} seconds.\n{2}".format(commands[0], timeout_sec, - _TIMEOUT_MSG)) - fatal_error(msg) - finish = time.time() - run_time_msg = "run time : {0:.2f} seconds".format(finish - start) - logging.info(run_time_msg) - output = proc.stdout.read() - log_process_output(output) - status = proc.returncode +def _hanging_msg(working_directory, command): + print(""" - # NOTE(bja, 2018-03) need to cleanup open files. In python3 use - # "with subprocess.Popen(...) as proc:", but that is not available - # with python2 unless we create a context manager. - proc.stdout.close() +Command '{command}' +from directory {working_directory} +has taken {hanging_sec} seconds. It may be hanging. - if status != 0: - raise subprocess.CalledProcessError(returncode=status, - cmd=commands, - output=output) - - if status_to_caller and output_to_caller: - ret_value = (status, output) - elif status_to_caller: - ret_value = status - elif output_to_caller: - ret_value = output - else: - ret_value = None +The command will continue to run, but you may want to abort +manage_externals with ^C and investigate. A possible cause of hangs is +when svn or git require authentication to access a private +repository. On some systems, svn and git requests for authentication +information will not be displayed to the user. In this case, the program +will appear to hang. Ensure you can run svn and git manually and access +all repositories without entering your authentication information. - return ret_value +""".format(command=command, + working_directory=working_directory, + hanging_sec=_HANGING_SEC)) def execute_subprocess(commands, status_to_caller=False, @@ -288,19 +240,24 @@ def execute_subprocess(commands, status_to_caller=False, return code, otherwise execute_subprocess treats non-zero return status as an error and raises an exception. - NOTE(bja, 2018-03) we are polling the running process to avoid - having it hang indefinitely if there is input that we don't - detect. Some large checkouts are multiple minutes long. For now we - are setting the timeout interval to five minutes. - """ - msg = 'In directory: {0}\nexecute_subprocess running command:'.format( - os.getcwd()) + cwd = os.getcwd() + msg = 'In directory: {0}\nexecute_subprocess running command:'.format(cwd) logging.info(msg) - logging.info(commands) + commands_str = ' '.join(commands) + logging.info(commands_str) + return_to_caller = status_to_caller or output_to_caller + status = -1 + output = '' + hanging_timer = Timer(_HANGING_SEC, _hanging_msg, + kwargs={"working_directory": cwd, + "command": commands_str}) + hanging_timer.start() try: - ret_value = _poll_subprocess( - commands, status_to_caller, output_to_caller) + output = subprocess.check_output(commands, stderr=subprocess.STDOUT, + universal_newlines=True) + log_process_output(output) + status = 0 except OSError as error: msg = failed_command_msg( 'Command execution failed. Does the executable exist?', @@ -319,20 +276,27 @@ def execute_subprocess(commands, status_to_caller=False, # simple status check. If returning, it is the callers # responsibility determine if an error occurred and handle it # appropriately. - if status_to_caller and output_to_caller: - ret_value = (error.returncode, error.output) - elif status_to_caller: - ret_value = error.returncode - elif output_to_caller: - ret_value = error.output - else: + if not return_to_caller: msg_context = ('Process did not run successfully; ' 'returned status {0}'.format(error.returncode)) msg = failed_command_msg(msg_context, commands, output=error.output) logging.error(error) + logging.error(msg) log_process_output(error.output) fatal_error(msg) + status = error.returncode + finally: + hanging_timer.cancel() + + if status_to_caller and output_to_caller: + ret_value = (status, output) + elif status_to_caller: + ret_value = status + elif output_to_caller: + ret_value = output + else: + ret_value = None return ret_value @@ -363,7 +327,4 @@ def failed_command_msg(msg_context, command, output=None): if output: errmsg += 'See above for output from failed command.\n' - errmsg += 'Please check the log file {log} for more details.'.format( - log=LOG_FILE_NAME) - return errmsg diff --git a/test/.pylint.rc b/test/.pylint.rc index 3e66113f..64abd03e 100644 --- a/test/.pylint.rc +++ b/test/.pylint.rc @@ -50,7 +50,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=bad-continuation +disable=bad-continuation,useless-object-inheritance # Enable the message, report, category or checker with the given id(s). You can diff --git a/test/Makefile b/test/Makefile index 8181552b..293e3607 100644 --- a/test/Makefile +++ b/test/Makefile @@ -39,7 +39,7 @@ PYLINT_ARGS=-j 2 --rcfile=.pylint.rc # code coverage COVERAGE=coverage -COVERAGE_ARGS=--rcfile=.coveragerc --append +COVERAGE_ARGS=--rcfile=.coveragerc # source files SRC = \ @@ -72,8 +72,8 @@ test : utest stest .PHONY : readme readme : $(CHECKOUT_EXE) printf "%s\n\n" "-- AUTOMATICALLY GENERATED FILE. DO NOT EDIT --" > $(README) - printf "%s" '[![Build Status](https://travis-ci.org/NCAR/manage_externals.svg?branch=master)](https://travis-ci.org/NCAR/manage_externals)' >> $(README) - printf "%s" '[![Coverage Status](https://coveralls.io/repos/github/NCAR/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/NCAR/manage_externals?branch=master)' >> $(README) + printf "%s" '[![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)' >> $(README) + printf "%s" '[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master)' >> $(README) printf "\n%s\n" '```' >> $(README) $(CHECKOUT_EXE) --help >> $(README) @@ -92,10 +92,13 @@ lint : FORCE stylint : style lint .PHONY : coverage +# Need to use a single coverage run with a single pattern rather than +# using two separate commands with separate patterns for test_unit_*.py +# and test_sys_*.py: The latter clobbers some results from the first +# run, even if we use the --append flag to 'coverage run'. coverage : FORCE $(PYPATH) $(COVERAGE) erase - $(PYPATH) $(COVERAGE) run $(COVERAGE_ARGS) $(TEST_ARGS) --pattern 'test_unit_*.py' - $(PYPATH) $(COVERAGE) run $(COVERAGE_ARGS) $(TEST_ARGS) --pattern 'test_sys_*.py' + $(PYPATH) $(COVERAGE) run $(COVERAGE_ARGS) $(TEST_ARGS) --pattern 'test_*.py' $(PYPATH) $(COVERAGE) html # diff --git a/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d b/test/repos/simple-ext.git/objects/14/2711fdbbcb8034d7cad6bae6801887b12fe61d new file mode 100644 index 0000000000000000000000000000000000000000..acaf7889b47c54ee0dea121c73d505ca14ad369b GIT binary patch literal 83 zcmV-Z0IdIb0ZYosPg1ZjWC+Q~ELKR%%t=)!&d4v#Nl{3x$Sf{V$jnnnRLILO%1z8s pNX|%2&dx6_QAh$}pz8eG%#xDS6o{JQg2bZYRJa;FE&z4gA7ySEC>H<# literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 b/test/repos/simple-ext.git/objects/60/7ec299c17dd285c029edc41a0109e49d441380 new file mode 100644 index 0000000000000000000000000000000000000000..3f6959cc54afd45fa4f64be25ccb3bb8580183c9 GIT binary patch literal 168 zcmV;Z09XHb0hNwR4#F@D1zG15z5poBzY;=l6vuV}(Nff=3vN&02JB|>MsIi;$9n_k z!>-M$Ac)DAYy~^^qUu9WLY{J}xkT>CQ3)XSx>0V^p=O;s>7G-EI{FfcPQQP4}zEXhpI%P&f0aFl&|Gw+GS!K3kZ)1Ezh zejs~i1S3>cQEFmJZmM2MMG3KlvCEtNF?@%PbVOT{Nm)vLb%0Bl_``r7C@umAu6 literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 b/test/repos/simple-ext.git/objects/d8/ed2f33179d751937f8fde2e33921e4827babf4 new file mode 100644 index 0000000000000000000000000000000000000000..f08ae820c9c89927f9898c5646134f7c519a6b04 GIT binary patch literal 60 zcmV-C0K@-y0V^p=O;s>4W-v4`Ff%bxC@xJ($t;Rb%gjmDE2$`95K$NWyZdy5$@Np$ Sc0Fs5Xy2&+OcnsW_!K#a0~pW% literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.git/refs/heads/master b/test/repos/simple-ext.git/refs/heads/master index 5c675049..adf1ccb0 100644 --- a/test/repos/simple-ext.git/refs/heads/master +++ b/test/repos/simple-ext.git/refs/heads/master @@ -1 +1 @@ -9b75494003deca69527bb64bcaa352e801611dd2 +607ec299c17dd285c029edc41a0109e49d441380 diff --git a/test/repos/simple-ext.git/refs/tags/tag2 b/test/repos/simple-ext.git/refs/tags/tag2 new file mode 100644 index 00000000..4160b6c4 --- /dev/null +++ b/test/repos/simple-ext.git/refs/tags/tag2 @@ -0,0 +1 @@ +b7692b6d391899680da7b9b6fd8af4c413f06fe7 diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 80feea88..df726f2b 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -42,6 +42,7 @@ from manic.externals_description import ExternalsDescription from manic.externals_description import DESCRIPTION_SECTION, VERSION_ITEM +from manic.externals_description import git_submodule_status from manic.externals_status import ExternalStatus from manic.repository_git import GitRepository from manic.utils import printlog, execute_subprocess @@ -83,9 +84,12 @@ CFG_NAME = 'externals.cfg' CFG_SUB_NAME = 'sub-externals.cfg' README_NAME = 'readme.txt' +REMOTE_BRANCH_FEATURE2 = 'feature2' SVN_TEST_REPO = 'https://github.com/escomp/cesm' +# Disable too-many-public-methods error +# pylint: disable=R0904 def setUpModule(): # pylint: disable=C0103 """Setup for all tests in this module. It is called once per module! @@ -130,7 +134,7 @@ def container_full(self, dest_dir): tag='tag1') self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch='feature2') + branch=REMOTE_BRANCH_FEATURE2) self.create_section(SIMPLE_REPO_NAME, 'simp_opt', tag='tag1', required=False) @@ -138,7 +142,7 @@ def container_full(self, dest_dir): self.create_section(MIXED_REPO_NAME, 'mixed_req', branch='master', externals=CFG_SUB_NAME) - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_required(self, dest_dir): """Create a container externals file with only simple externals. @@ -149,12 +153,12 @@ def container_simple_required(self, dest_dir): tag='tag1') self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch='feature2') + branch=REMOTE_BRANCH_FEATURE2) self.create_section(SIMPLE_REPO_NAME, 'simp_hash', ref_hash='60b1cc1a38d63') - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_optional(self, dest_dir): """Create a container externals file with optional simple externals @@ -167,7 +171,7 @@ def container_simple_optional(self, dest_dir): self.create_section(SIMPLE_REPO_NAME, 'simp_opt', tag='tag1', required=False) - self._write_config(dest_dir) + self.write_config(dest_dir) def container_simple_svn(self, dest_dir): """Create a container externals file with only simple externals. @@ -179,7 +183,26 @@ def container_simple_svn(self, dest_dir): self.create_svn_external('svn_branch', branch='trunk') self.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') - self._write_config(dest_dir) + self.write_config(dest_dir) + + def container_sparse(self, dest_dir): + """Create a container with a full external and a sparse external + + """ + # Create a file for a sparse pattern match + sparse_filename = 'sparse_checkout' + with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: + sfile.write('readme.txt') + + self.create_config() + self.create_section(SIMPLE_REPO_NAME, 'simp_tag', + tag='tag2') + + sparse_relpath = '../../{}'.format(sparse_filename) + self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self.write_config(dest_dir) def mixed_simple_base(self, dest_dir): """Create a mixed-use base externals file with only simple externals. @@ -191,12 +214,12 @@ def mixed_simple_base(self, dest_dir): tag='tag1') self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch='feature2') + branch=REMOTE_BRANCH_FEATURE2) self.create_section(SIMPLE_REPO_NAME, 'simp_hash', ref_hash='60b1cc1a38d63') - self._write_config(dest_dir) + self.write_config(dest_dir) def mixed_simple_sub(self, dest_dir): """Create a mixed-use sub externals file with only simple externals. @@ -207,11 +230,12 @@ def mixed_simple_sub(self, dest_dir): tag='tag1', path=SUB_EXTERNALS_PATH) self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch='feature2', path=SUB_EXTERNALS_PATH) + branch=REMOTE_BRANCH_FEATURE2, + path=SUB_EXTERNALS_PATH) - self._write_config(dest_dir, filename=CFG_SUB_NAME) + self.write_config(dest_dir, filename=CFG_SUB_NAME) - def _write_config(self, dest_dir, filename=CFG_NAME): + def write_config(self, dest_dir, filename=CFG_NAME): """Write the configuration file to disk """ @@ -235,22 +259,41 @@ def create_metadata(self): self._schema_version) def create_section(self, repo_type, name, tag='', branch='', - ref_hash='', - required=True, path=EXTERNALS_NAME, externals=''): + ref_hash='', required=True, path=EXTERNALS_NAME, + externals='', repo_path=None, from_submodule=False, + sparse=''): + # pylint: disable=too-many-branches """Create a config section with autofilling some items and handling optional items. """ # pylint: disable=R0913 self._config.add_section(name) - self._config.set(name, ExternalsDescription.PATH, - os.path.join(path, name)) + if not from_submodule: + self._config.set(name, ExternalsDescription.PATH, + os.path.join(path, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_GIT) - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) - self._config.set(name, ExternalsDescription.REPO_URL, repo_url) + # from_submodules is incompatible with some other options, turn them off + if (from_submodule and + ((repo_path is not None) or tag or ref_hash or branch)): + printlog('create_section: "from_submodule" is incompatible with ' + '"repo_url", "tag", "hash", and "branch" options;\n' + 'Ignoring those options for {}'.format(name)) + repo_url = None + tag = '' + ref_hash = '' + branch = '' + + if repo_path is not None: + repo_url = repo_path + else: + repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) + + if not from_submodule: + self._config.set(name, ExternalsDescription.REPO_URL, repo_url) self._config.set(name, ExternalsDescription.REQUIRED, str(required)) @@ -266,6 +309,12 @@ def create_section(self, repo_type, name, tag='', branch='', if externals: self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sparse: + self._config.set(name, ExternalsDescription.SPARSE, sparse) + + if from_submodule: + self._config.set(name, ExternalsDescription.SUBMODULE, "True") + def create_section_ext_only(self, name, required=True, externals=CFG_SUB_NAME): """Create a config section with autofilling some items and handling @@ -329,6 +378,31 @@ def create_branch(dest_dir, repo_name, branch, with_commit=False): execute_subprocess(cmd) os.chdir(cwd) + @staticmethod + def create_commit(dest_dir, repo_name, local_tracking_branch=None): + """Make a commit on whatever is currently checked out. + + This is used to test sync state changes from local commits on + detached heads and tracking branches. + + """ + cwd = os.getcwd() + repo_root = os.path.join(dest_dir, EXTERNALS_NAME) + repo_root = os.path.join(repo_root, repo_name) + os.chdir(repo_root) + if local_tracking_branch: + cmd = ['git', 'checkout', '-b', local_tracking_branch, ] + execute_subprocess(cmd) + + msg = 'work on great new feature!' + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) + def update_branch(self, dest_dir, name, branch, repo_type=None, filename=CFG_NAME): """Update a repository branch, and potentially the remote. @@ -350,7 +424,7 @@ def update_branch(self, dest_dir, name, branch, repo_type=None, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): """Update a repository branch, and potentially the remote. @@ -364,7 +438,7 @@ def update_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_tag(self, dest_dir, name, tag, repo_type=None, filename=CFG_NAME, remove_branch=True): @@ -389,7 +463,7 @@ def update_tag(self, dest_dir, name, tag, repo_type=None, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_underspecify_branch_tag(self, dest_dir, name, filename=CFG_NAME): @@ -408,7 +482,7 @@ def update_underspecify_branch_tag(self, dest_dir, name, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_underspecify_remove_url(self, dest_dir, name, filename=CFG_NAME): @@ -421,7 +495,7 @@ def update_underspecify_remove_url(self, dest_dir, name, except BaseException: pass - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) def update_protocol(self, dest_dir, name, protocol, repo_type=None, filename=CFG_NAME): @@ -434,7 +508,7 @@ def update_protocol(self, dest_dir, name, protocol, repo_type=None, repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_type) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) - self._write_config(dest_dir, filename) + self.write_config(dest_dir, filename) class BaseTestSysCheckout(unittest.TestCase): @@ -486,7 +560,7 @@ def tearDown(self): # return to our common starting point os.chdir(self._return_dir) - def setup_test_repo(self, parent_repo_name): + def setup_test_repo(self, parent_repo_name, dest_dir_in=None): """Setup the paths and clone the base test repo """ @@ -495,8 +569,12 @@ def setup_test_repo(self, parent_repo_name): print("Test repository name: {0}".format(test_dir_name)) parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - test_dir_name) + if dest_dir_in is None: + dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], + test_dir_name) + else: + dest_dir = dest_dir_in + # pylint: disable=W0212 GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) return dest_dir @@ -601,6 +679,10 @@ def _check_simple_tag_dirty(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_tag'.format(directory) self._check_generic_ok_dirty_required(tree, name) + def _check_simple_tag_modified(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_tag'.format(directory) + self._check_generic_modified_ok_required(tree, name) + def _check_simple_branch_empty(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_branch'.format(directory) self._check_generic_empty_default_required(tree, name) @@ -621,6 +703,10 @@ def _check_simple_hash_ok(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_hash'.format(directory) self._check_generic_ok_clean_required(tree, name) + def _check_simple_hash_modified(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_hash'.format(directory) + self._check_generic_modified_ok_required(tree, name) + def _check_simple_req_empty(self, tree, directory=EXTERNALS_NAME): name = './{0}/simp_req'.format(directory) self._check_generic_empty_default_required(tree, name) @@ -649,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): name = './{0}/mixed_req'.format(directory) self._check_generic_modified_ok_required(tree, name) + def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_empty_default_required(tree, name) + + def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): + name = './{0}/simp_sparse'.format(directory) + self._check_generic_ok_clean_required(tree, name) + # ---------------------------------------------------------------- # # Check results for groups of externals under specific conditions @@ -673,6 +767,12 @@ def _check_container_simple_required_post_checkout(self, overall, tree): self._check_simple_branch_ok(tree) self._check_simple_hash_ok(tree) + def _check_container_simple_required_out_of_sync(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_modified(tree) + self._check_simple_branch_modified(tree) + self._check_simple_hash_modified(tree) + def _check_container_simple_optional_pre_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_req_empty(tree) @@ -711,6 +811,18 @@ def _check_container_full_pre_checkout(self, overall, tree): self._check_simple_opt_empty(tree) self._check_mixed_ext_branch_required_pre_checkout(overall, tree) + def _check_container_component_post_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_opt_ok(tree) + self._check_simple_tag_empty(tree) + self._check_simple_branch_empty(tree) + + def _check_container_component_post_checkout2(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_opt_ok(tree) + self._check_simple_tag_empty(tree) + self._check_simple_branch_ok(tree) + def _check_container_full_post_checkout(self, overall, tree): self.assertEqual(overall, 0) self._check_simple_tag_ok(tree) @@ -791,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) + def _check_container_sparse_pre_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_empty(tree) + self._check_simple_sparse_empty(tree) + + def _check_container_sparse_post_checkout(self, overall, tree): + self.assertEqual(overall, 0) + self._check_simple_tag_ok(tree) + self._check_simple_sparse_ok(tree) + + def _check_file_exists(self, repo_dir, pathname): + "Check that exists in " + self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) + + def _check_file_absent(self, repo_dir, pathname): + "Check that does not exist in " + self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals @@ -941,6 +1070,47 @@ def test_container_simple_untracked(self): self.status_args) self._check_container_simple_required_post_checkout(overall, tree) + def test_container_simple_detached_sync(self): + """Verify that a container with simple subrepos generates the correct + out of sync status when making commits from a detached head + state. + + """ + # create repo + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + self._generator.container_simple_required(under_test_dir) + + # status of empty repo + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_simple_required_pre_checkout(overall, tree) + + # checkout + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.checkout_args) + self._check_container_simple_required_checkout(overall, tree) + + # make a commit on the detached head of the tag and hash externals + self._generator.create_commit(under_test_dir, 'simp_tag') + self._generator.create_commit(under_test_dir, 'simp_hash') + self._generator.create_commit(under_test_dir, 'simp_branch') + + # status of repo, branch, tag and hash should all be out of sync! + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_simple_required_out_of_sync(overall, tree) + + # checkout + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.checkout_args) + # same pre-checkout out of sync status + self._check_container_simple_required_out_of_sync(overall, tree) + + # now status should be in-sync + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_simple_required_post_checkout(overall, tree) + def test_container_remote_branch(self): """Verify that a container with remote branch change works @@ -957,7 +1127,7 @@ def test_container_remote_branch(self): # update the config file to point to a different remote with # the same branch self._generator.update_branch(under_test_dir, 'simp_branch', - 'feature2', SIMPLE_FORK_NAME) + REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) # status of simp_branch should be out of sync overall, tree = self.execute_cmd_in_dir(under_test_dir, @@ -1066,7 +1236,7 @@ def test_container_preserve_dot(self): # update the config file to point to a different remote with # the same branch self._generator.update_branch(under_test_dir, 'simp_branch', - 'feature2', SIMPLE_FORK_NAME) + REMOTE_BRANCH_FEATURE2, SIMPLE_FORK_NAME) # checkout overall, tree = self.execute_cmd_in_dir(under_test_dir, self.checkout_args) @@ -1114,6 +1284,14 @@ def test_container_full(self): self.status_args) self._check_container_full_post_checkout(overall, tree) + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + # update the mixed-use repo to point to different branch self._generator.update_branch(under_test_dir, 'mixed_req', 'new-feature', MIXED_REPO_NAME) @@ -1137,6 +1315,38 @@ def test_container_full(self): self.status_args) self._check_container_full_post_checkout(overall, tree) + def test_container_component(self): + """Verify that optional component checkout works + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_full(under_test_dir) + + # inital checkout, first try a nonexistant component argument noref + checkout_args = ['simp_opt', 'noref'] + checkout_args.extend(self.checkout_args) + + with self.assertRaises(RuntimeError): + self.execute_cmd_in_dir(under_test_dir, checkout_args) + + checkout_args = ['simp_opt'] + checkout_args.extend(self.checkout_args) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + checkout_args) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_component_post_checkout(overall, tree) + checkout_args.append('simp_branch') + overall, tree = self.execute_cmd_in_dir(under_test_dir, + checkout_args) + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_component_post_checkout2(overall, tree) + def test_mixed_simple(self): """Verify that a mixed use repo can serve as a 'full' container, pulling in a set of externals and a seperate set of sub-externals. @@ -1162,6 +1372,40 @@ def test_mixed_simple(self): self.status_args) self._check_mixed_cont_simple_required_post_checkout(overall, tree) + def test_container_sparse(self): + """Verify that 'full' container with simple subrepo + can run a sparse checkout and generate the correct initial status. + + """ + # create the test repository + under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME) + + # create the top level externals file + self._generator.container_sparse(under_test_dir) + + # inital checkout + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.checkout_args) + self._check_container_sparse_pre_checkout(overall, tree) + + overall, tree = self.execute_cmd_in_dir(under_test_dir, + self.status_args) + self._check_container_sparse_post_checkout(overall, tree) + + # Check existance of some files + subrepo_path = os.path.join('externals', 'simp_tag') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_exists(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + subrepo_path = os.path.join('externals', 'simp_sparse') + self._check_file_exists(under_test_dir, + os.path.join(subrepo_path, 'readme.txt')) + self._check_file_absent(under_test_dir, os.path.join(subrepo_path, + 'simple_subdir', + 'subdir_file.txt')) + class TestSysCheckoutSVN(BaseTestSysCheckout): """Run systems level tests of checkout_externals accessing svn repositories @@ -1308,6 +1552,237 @@ def test_container_simple_svn(self): self.verbose_args) self._check_container_simple_svn_post_checkout(overall, tree) +class TestSubrepoCheckout(BaseTestSysCheckout): + # Need to store information at setUp time for checking + # pylint: disable=too-many-instance-attributes + """Run tests to ensure proper handling of repos with submodules. + + By default, submodules in git repositories are checked out. A git + repository checked out as a submodule is treated as if it was + listed in an external with the same properties as in the source + .gitmodules file. + """ + + def setUp(self): + """Setup for all submodule checkout tests + Create a repo with two submodule repositories. + """ + + # Run the basic setup + super(TestSubrepoCheckout, self).setUp() + # create test repo + # We need to do this here (rather than have a static repo) because + # git submodules do not allow for variables in .gitmodules files + self._test_repo_name = 'test_repo_with_submodules' + self._bare_branch_name = 'subrepo_branch' + self._config_branch_name = 'subrepo_config_branch' + self._container_extern_name = 'externals_container.cfg' + self._my_test_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], + self._test_id) + self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) + self._checkout_dir = 'repo_with_submodules' + check_dir = self.setup_test_repo(CONTAINER_REPO_NAME, + dest_dir_in=self._repo_dir) + self.assertTrue(self._repo_dir == check_dir) + # Add the submodules + cwd = os.getcwd() + fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_NAME) + simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO_NAME) + self._simple_ext_fork_name = SIMPLE_FORK_NAME.split('.')[0] + self._simple_ext_name = SIMPLE_REPO_NAME.split('.')[0] + os.chdir(self._repo_dir) + # Add a branch with a subrepo + cmd = ['git', 'branch', self._bare_branch_name, 'master'] + execute_subprocess(cmd) + cmd = ['git', 'checkout', self._bare_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'submodule', 'add', fork_repo_dir] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-am', "'Added simple-ext-fork as a submodule'"] + execute_subprocess(cmd) + # Save the fork repo hash for comparison + os.chdir(self._simple_ext_fork_name) + self._fork_hash_check = self.get_git_hash() + os.chdir(self._repo_dir) + # Now, create a branch to test from_sbmodule + cmd = ['git', 'branch', + self._config_branch_name, self._bare_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'checkout', self._config_branch_name] + execute_subprocess(cmd) + cmd = ['git', 'submodule', 'add', simple_repo_dir] + execute_subprocess(cmd) + # Checkout feature2 + os.chdir(self._simple_ext_name) + cmd = ['git', 'branch', 'feature2', 'origin/feature2'] + execute_subprocess(cmd) + cmd = ['git', 'checkout', 'feature2'] + execute_subprocess(cmd) + # Save the fork repo hash for comparison + self._simple_hash_check = self.get_git_hash() + os.chdir(self._repo_dir) + self.create_externals_file(filename=self._container_extern_name, + dest_dir=self._repo_dir, from_submodule=True) + cmd = ['git', 'add', self._container_extern_name] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-am', "'Added simple-ext as a submodule'"] + execute_subprocess(cmd) + # Reset to master + cmd = ['git', 'checkout', 'master'] + execute_subprocess(cmd) + os.chdir(cwd) + + @staticmethod + def get_git_hash(revision="HEAD"): + """Return the hash for """ + cmd = ['git', 'rev-parse', revision] + git_out = execute_subprocess(cmd, output_to_caller=True) + return git_out.strip() + + def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, + branch_name=None, sub_externals=None, + from_submodule=False): + # pylint: disable=too-many-arguments + """Create a container externals file with only simple externals. + + """ + self._generator.create_config() + + if dest_dir is None: + dest_dir = self._my_test_dir + + if from_submodule: + self._generator.create_section(SIMPLE_FORK_NAME, + self._simple_ext_fork_name, + from_submodule=True) + self._generator.create_section(SIMPLE_REPO_NAME, + self._simple_ext_name, + branch='feature3', path='', + from_submodule=False) + else: + if branch_name is None: + branch_name = 'master' + + self._generator.create_section(self._test_repo_name, + self._checkout_dir, + branch=branch_name, + path=name, externals=sub_externals, + repo_path=self._repo_dir) + + self._generator.write_config(dest_dir, filename=filename) + + def idempotence_check(self, checkout_dir): + """Verify that calling checkout_externals and + checkout_externals --status does not cause errors""" + cwd = os.getcwd() + os.chdir(checkout_dir) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.status_args) + self.assertTrue(overall == 0) + os.chdir(cwd) + + def test_submodule_checkout_bare(self): + """Verify that a git repo with submodule is properly checked out + This test if for where there is no 'externals' keyword in the + parent repo. + Correct behavior is that the submodule is checked out using + normal git submodule behavior. + """ + simple_ext_fork_tag = "(tag1)" + simple_ext_fork_status = " " + self.create_externals_file(branch_name=self._bare_branch_name) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertTrue(os.path.exists(fork_file)) + os.chdir(checkout_dir) + submods = git_submodule_status(checkout_dir) + self.assertEqual(len(submods.keys()), 1) + self.assertTrue(self._simple_ext_fork_name in submods) + submod = submods[self._simple_ext_fork_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._fork_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], simple_ext_fork_status) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], simple_ext_fork_tag) + os.chdir(cwd) + self.idempotence_check(checkout_dir) + + def test_submodule_checkout_none(self): + """Verify that a git repo with submodule is properly checked out + This test is for when 'externals=None' is in parent repo's + externals cfg file. + Correct behavior is the submodle is not checked out. + """ + self.create_externals_file(branch_name=self._bare_branch_name, + sub_externals="none") + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertFalse(os.path.exists(fork_file)) + os.chdir(cwd) + self.idempotence_check(checkout_dir) + + def test_submodule_checkout_config(self): # pylint: disable=too-many-locals + """Verify that a git repo with submodule is properly checked out + This test if for when the 'from_submodule' keyword is used in the + parent repo. + Correct behavior is that the submodule is checked out using + normal git submodule behavior. + """ + tag_check = None # Not checked out as submodule + status_check = "-" # Not checked out as submodule + self.create_externals_file(branch_name=self._config_branch_name, + sub_externals=self._container_extern_name) + overall, _ = self.execute_cmd_in_dir(self._my_test_dir, + self.checkout_args) + self.assertTrue(overall == 0) + cwd = os.getcwd() + checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) + fork_file = os.path.join(checkout_dir, + self._simple_ext_fork_name, "readme.txt") + self.assertTrue(os.path.exists(fork_file)) + os.chdir(checkout_dir) + # Check submodule status + submods = git_submodule_status(checkout_dir) + self.assertEqual(len(submods.keys()), 2) + self.assertTrue(self._simple_ext_fork_name in submods) + submod = submods[self._simple_ext_fork_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._fork_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], status_check) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], tag_check) + self.assertTrue(self._simple_ext_name in submods) + submod = submods[self._simple_ext_name] + self.assertTrue('hash' in submod) + self.assertEqual(submod['hash'], self._simple_hash_check) + self.assertTrue('status' in submod) + self.assertEqual(submod['status'], status_check) + self.assertTrue('tag' in submod) + self.assertEqual(submod['tag'], tag_check) + # Check fork repo status + os.chdir(self._simple_ext_fork_name) + self.assertEqual(self.get_git_hash(), self._fork_hash_check) + os.chdir(checkout_dir) + os.chdir(self._simple_ext_name) + hash_check = self.get_git_hash('origin/feature3') + self.assertEqual(self.get_git_hash(), hash_check) + os.chdir(cwd) + self.idempotence_check(checkout_dir) class TestSysCheckoutErrors(BaseTestSysCheckout): """Run systems level tests of error conditions in checkout_externals diff --git a/test/test_sys_repository_git.py b/test/test_sys_repository_git.py new file mode 100644 index 00000000..f6dbf842 --- /dev/null +++ b/test/test_sys_repository_git.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python + +"""Tests of some of the functionality in repository_git.py that actually +interacts with git repositories. + +We're calling these "system" tests because we expect them to be a lot +slower than most of the unit tests. + +""" + +from __future__ import absolute_import +from __future__ import unicode_literals +from __future__ import print_function + +import os +import shutil +import tempfile +import unittest + +from manic.repository_git import GitRepository +from manic.externals_description import ExternalsDescription +from manic.externals_description import ExternalsDescriptionDict +from manic.utils import execute_subprocess + +# NOTE(wjs, 2018-04-09) I find a mix of camel case and underscores to be +# more readable for unit test names, so I'm disabling pylint's naming +# convention check +# pylint: disable=C0103 + +# Allow access to protected members +# pylint: disable=W0212 + + +class GitTestCase(unittest.TestCase): + """Adds some git-specific unit test functionality on top of TestCase""" + + def assertIsHash(self, maybe_hash): + """Assert that the string given by maybe_hash really does look + like a git hash. + """ + + # Ensure it is non-empty + self.assertTrue(maybe_hash, msg="maybe_hash is empty") + + # Ensure it has a single string + self.assertEqual(1, len(maybe_hash.split()), + msg="maybe_hash has multiple strings: {}".format(maybe_hash)) + + # Ensure that the only characters in the string are ones allowed + # in hashes + allowed_chars_set = set('0123456789abcdef') + self.assertTrue(set(maybe_hash) <= allowed_chars_set, + msg="maybe_hash has non-hash characters: {}".format(maybe_hash)) + + +class TestGitTestCase(GitTestCase): + """Tests GitTestCase""" + + def test_assertIsHash_true(self): + """Ensure that assertIsHash passes for something that looks + like a hash""" + self.assertIsHash('abc123') + + def test_assertIsHash_empty(self): + """Ensure that assertIsHash raises an AssertionError for an + empty string""" + with self.assertRaises(AssertionError): + self.assertIsHash('') + + def test_assertIsHash_multipleStrings(self): + """Ensure that assertIsHash raises an AssertionError when + given multiple strings""" + with self.assertRaises(AssertionError): + self.assertIsHash('abc123 def456') + + def test_assertIsHash_badChar(self): + """Ensure that assertIsHash raises an AssertionError when given a + string that has a character that doesn't belong in a hash + """ + with self.assertRaises(AssertionError): + self.assertIsHash('abc123g') + + +class TestGitRepositoryGitCommands(GitTestCase): + """Test some git commands in RepositoryGit + + It's silly that we need to create a repository in order to test + these git commands. Much or all of the git functionality that is + currently in repository_git.py should eventually be moved to a + separate module that is solely responsible for wrapping git + commands; that would allow us to test it independently of this + repository class. + """ + + # ======================================================================== + # Test helper functions + # ======================================================================== + + def setUp(self): + # directory we want to return to after the test system and + # checkout_externals are done cd'ing all over the place. + self._return_dir = os.getcwd() + + self._tmpdir = tempfile.mkdtemp() + os.chdir(self._tmpdir) + + self._name = 'component' + rdata = {ExternalsDescription.PROTOCOL: 'git', + ExternalsDescription.REPO_URL: + '/path/to/local/repo', + ExternalsDescription.TAG: + 'tag1', + } + + data = {self._name: + { + ExternalsDescription.REQUIRED: False, + ExternalsDescription.PATH: 'junk', + ExternalsDescription.EXTERNALS: '', + ExternalsDescription.REPO: rdata, + }, + } + model = ExternalsDescriptionDict(data) + repo = model[self._name][ExternalsDescription.REPO] + self._repo = GitRepository('test', repo) + + def tearDown(self): + # return to our common starting point + os.chdir(self._return_dir) + + shutil.rmtree(self._tmpdir, ignore_errors=True) + + @staticmethod + def make_git_repo(): + """Turn the current directory into an empty git repository""" + execute_subprocess(['git', 'init']) + + @staticmethod + def add_git_commit(): + """Add a git commit in the current directory""" + with open('README', 'a') as myfile: + myfile.write('more info') + execute_subprocess(['git', 'add', 'README']) + execute_subprocess(['git', 'commit', '-m', 'my commit message']) + + @staticmethod + def checkout_git_branch(branchname): + """Checkout a new branch in the current directory""" + execute_subprocess(['git', 'checkout', '-b', branchname]) + + @staticmethod + def make_git_tag(tagname): + """Make a lightweight tag at the current commit""" + execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) + + @staticmethod + def checkout_ref(refname): + """Checkout the given refname in the current directory""" + execute_subprocess(['git', 'checkout', refname]) + + # ======================================================================== + # Begin actual tests + # ======================================================================== + + def test_currentHash_returnsHash(self): + """Ensure that the _git_current_hash function returns a hash""" + self.make_git_repo() + self.add_git_commit() + hash_found, myhash = self._repo._git_current_hash() + self.assertTrue(hash_found) + self.assertIsHash(myhash) + + def test_currentHash_outsideGitRepo(self): + """Ensure that the _git_current_hash function returns False when + outside a git repository""" + hash_found, myhash = self._repo._git_current_hash() + self.assertFalse(hash_found) + self.assertEqual('', myhash) + + def test_currentBranch_onBranch(self): + """Ensure that the _git_current_branch function returns the name + of the branch""" + self.make_git_repo() + self.add_git_commit() + self.checkout_git_branch('foo') + branch_found, mybranch = self._repo._git_current_branch() + self.assertTrue(branch_found) + self.assertEqual('foo', mybranch) + + def test_currentBranch_notOnBranch(self): + """Ensure that the _git_current_branch function returns False + when not on a branch""" + self.make_git_repo() + self.add_git_commit() + self.make_git_tag('mytag') + self.checkout_ref('mytag') + branch_found, mybranch = self._repo._git_current_branch() + self.assertFalse(branch_found) + self.assertEqual('', mybranch) + + def test_currentBranch_outsideGitRepo(self): + """Ensure that the _git_current_branch function returns False + when outside a git repository""" + branch_found, mybranch = self._repo._git_current_branch() + self.assertFalse(branch_found) + self.assertEqual('', mybranch) + + def test_currentTag_onTag(self): + """Ensure that the _git_current_tag function returns the name of + the tag""" + self.make_git_repo() + self.add_git_commit() + self.make_git_tag('some_tag') + tag_found, mytag = self._repo._git_current_tag() + self.assertTrue(tag_found) + self.assertEqual('some_tag', mytag) + + def test_currentTag_notOnTag(self): + """Ensure tha the _git_current_tag function returns False when + not on a tag""" + self.make_git_repo() + self.add_git_commit() + self.make_git_tag('some_tag') + self.add_git_commit() + tag_found, mytag = self._repo._git_current_tag() + self.assertFalse(tag_found) + self.assertEqual('', mytag) + + def test_currentTag_outsideGitRepo(self): + """Ensure that the _git_current_tag function returns False when + outside a git repository""" + tag_found, mytag = self._repo._git_current_tag() + self.assertFalse(tag_found) + self.assertEqual('', mytag) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_unit_externals_description.py b/test/test_unit_externals_description.py index 5de60e4f..637f760e 100644 --- a/test/test_unit_externals_description.py +++ b/test/test_unit_externals_description.py @@ -316,11 +316,13 @@ def setUp(self): """Create config object used as basis for all tests """ self._config = config_parser() + self._gmconfig = config_parser() self.setup_config() def setup_config(self): """Boiler plate construction of xml string for componet 1 """ + # Create a standard externals config with a single external name = 'test' self._config.add_section(name) self._config.set(name, ExternalsDescription.PATH, 'externals') @@ -332,6 +334,14 @@ def setup_config(self): self._config.add_section(DESCRIPTION_SECTION) self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0') + # Create a .gitmodules test + name = 'submodule "gitmodules_test"' + self._gmconfig.add_section(name) + self._gmconfig.set(name, "path", 'externals/test') + self._gmconfig.set(name, "url", '/path/to/repo') + # NOTE(goldy, 2019-03) Should test other possible keywords such as + # fetchRecurseSubmodules, ignore, and shallow + def test_cfg_v1_ok(self): """Test that a correct cfg v1 object is created by create_externals_description @@ -356,7 +366,7 @@ def test_dict(self): rdata = {ExternalsDescription.PROTOCOL: 'git', ExternalsDescription.REPO_URL: '/path/to/repo', ExternalsDescription.TAG: 'tagv1', - } + } desc = { 'test': { diff --git a/test/test_unit_repository.py b/test/test_unit_repository.py index 2152503c..5b9c242f 100644 --- a/test/test_unit_repository.py +++ b/test/test_unit_repository.py @@ -36,7 +36,8 @@ def setUp(self): ExternalsDescription.REPO_URL: 'junk_root', ExternalsDescription.TAG: 'junk_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } def test_create_repo_git(self): """Verify that several possible names for the 'git' protocol @@ -95,7 +96,8 @@ def test_tag(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.TAG: tag, ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.tag(), tag) @@ -112,7 +114,8 @@ def test_branch(self): ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.branch(), branch) @@ -125,11 +128,13 @@ def test_hash(self): protocol = 'test_protocol' url = 'test_url' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: EMPTY_STR, ExternalsDescription.TAG: EMPTY_STR, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } repo = Repository(name, repo_info) print(repo.__dict__) self.assertEqual(repo.hash(), ref) @@ -146,11 +151,13 @@ def test_tag_branch(self): branch = 'test_branch' tag = 'test_tag' ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -165,11 +172,13 @@ def test_tag_branch_hash(self): branch = 'test_branch' tag = 'test_tag' ref = 'deadc0de' + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) @@ -184,11 +193,13 @@ def test_no_tag_no_branch(self): branch = EMPTY_STR tag = EMPTY_STR ref = EMPTY_STR + sparse = EMPTY_STR repo_info = {ExternalsDescription.PROTOCOL: protocol, ExternalsDescription.REPO_URL: url, ExternalsDescription.BRANCH: branch, ExternalsDescription.TAG: tag, - ExternalsDescription.HASH: ref, } + ExternalsDescription.HASH: ref, + ExternalsDescription.SPARSE: sparse, } with self.assertRaises(RuntimeError): Repository(name, repo_info) diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index d481995d..4a0a334b 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -14,7 +14,6 @@ import os import shutil -import string import unittest from manic.repository_git import GitRepository @@ -23,57 +22,6 @@ from manic.externals_description import ExternalsDescriptionDict from manic.global_constants import EMPTY_STR -# pylint: disable=C0103 -GIT_BRANCH_OUTPUT_DETACHED_BRANCH_v1_8 = ''' -* (detached from origin/feature2) 36418b4 Work on feature2 - master 9b75494 [origin/master] Initialize repository. -''' -# pylint: enable=C0103 - - -GIT_BRANCH_OUTPUT_DETACHED_BRANCH = ''' -* (HEAD detached at origin/feature-2) 36418b4 Work on feature-2 - feature-2 36418b4 [origin/feature-2] Work on feature-2 - feature3 36418b4 Work on feature-2 - master 9b75494 [origin/master] Initialize repository. -''' - -GIT_BRANCH_OUTPUT_DETACHED_HASH = ''' -* (HEAD detached at 36418b4) 36418b4 Work on feature-2 - feature-2 36418b4 [origin/feature-2] Work on feature-2 - feature3 36418b4 Work on feature-2 - master 9b75494 [origin/master] Initialize repository. -''' - -GIT_BRANCH_OUTPUT_DETACHED_TAG = ''' -* (HEAD detached at tag1) 9b75494 Initialize repository. - feature-2 36418b4 [origin/feature-2] Work on feature-2 - feature3 36418b4 Work on feature-2 - master 9b75494 [origin/master] Initialize repository. -''' - -GIT_BRANCH_OUTPUT_UNTRACKED_BRANCH = ''' - feature-2 36418b4 [origin/feature-2] Work on feature-2 -* feature3 36418b4 Work on feature-2 - master 9b75494 [origin/master] Initialize repository. -''' - -GIT_BRANCH_OUTPUT_TRACKING_BRANCH = ''' -* feature-2 36418b4 [origin/feature-2] Work on feature-2 - feature3 36418b4 Work on feature-2 - master 9b75494 [origin/master] Initialize repository. -''' - -GIT_BRANCH_OUTPUT_TRACKING_BRANCH_AHEAD_BEHIND = ''' -* master 408a8920 [origin/master: ahead 3, behind 2] more junk - feature3 36418b4 Work on feature-2 -''' - -GIT_BRANCH_OUTPUT_TRACKING_BRANCH_AHEAD = ''' -* master 408a8920 [origin/master: ahead 3] more junk - feature3 36418b4 Work on feature-2 -''' - # NOTE(bja, 2017-11) order is important here. origin should be a # subset of other to trap errors on processing remotes! GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM = ''' @@ -86,8 +34,8 @@ ''' -class TestGitRepositoryCurrentRefBranch(unittest.TestCase): - """test the current_ref_from_branch_command on a git repository +class TestGitRepositoryCurrentRef(unittest.TestCase): + """test the current_ref command on a git repository """ def setUp(self): @@ -112,94 +60,80 @@ def setUp(self): repo = model[self._name][ExternalsDescription.REPO] self._repo = GitRepository('test', repo) - def test_ref_detached_from_tag(self): - """Test that we correctly identify that the ref is detached from a tag - """ - git_output = GIT_BRANCH_OUTPUT_DETACHED_TAG - expected = self._repo.tag() - result = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(result, expected) - - def test_ref_detached_hash(self): - """Test that we can identify ref is detached from a hash - - """ - git_output = GIT_BRANCH_OUTPUT_DETACHED_HASH - expected = '36418b4' - result = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(result, expected) - - def test_ref_detached_branch(self): - """Test that we can identify ref is detached from a remote branch - - """ - git_output = GIT_BRANCH_OUTPUT_DETACHED_BRANCH - expected = 'origin/feature-2' - result = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(result, expected) - - def test_ref_detached_branch_v1_8(self): - """Test that we can identify ref is detached from a remote branch + # + # mock methods replacing git system calls + # + @staticmethod + def _git_current_branch(branch_found, branch_name): + """Return a function that takes the place of + repo._git_current_branch, which returns the given output.""" + def my_git_current_branch(): + """mock function that can take the place of repo._git_current_branch""" + return branch_found, branch_name + return my_git_current_branch - """ - git_output = GIT_BRANCH_OUTPUT_DETACHED_BRANCH_v1_8 - expected = 'origin/feature2' - result = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(result, expected) + @staticmethod + def _git_current_tag(tag_found, tag_name): + """Return a function that takes the place of + repo._git_current_tag, which returns the given output.""" + def my_git_current_tag(): + """mock function that can take the place of repo._git_current_tag""" + return tag_found, tag_name + return my_git_current_tag - def test_ref_tracking_branch(self): - """Test that we correctly identify we are on a tracking branch - """ - git_output = GIT_BRANCH_OUTPUT_TRACKING_BRANCH - expected = 'origin/feature-2' - result = self._repo._current_ref_from_branch_command( - git_output) + @staticmethod + def _git_current_hash(hash_found, hash_name): + """Return a function that takes the place of + repo._git_current_hash, which returns the given output.""" + def my_git_current_hash(): + """mock function that can take the place of repo._git_current_hash""" + return hash_found, hash_name + return my_git_current_hash + + # ------------------------------------------------------------------------ + # Begin tests + # ------------------------------------------------------------------------ + + def test_ref_branch(self): + """Test that we correctly identify we are on a branch + """ + self._repo._git_current_branch = self._git_current_branch( + True, 'feature3') + self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') + self._repo._git_current_hash = self._git_current_hash(True, 'abc123') + expected = 'feature3' + result = self._repo._current_ref() self.assertEqual(result, expected) - def test_ref_tracking_branch_ahead(self): - """Test that we correctly identify we are on a tracking branch that is - ahead or behind the remote branch. - + def test_ref_detached_tag(self): + """Test that we correctly identify that the ref is detached at a tag """ - git_output = GIT_BRANCH_OUTPUT_TRACKING_BRANCH_AHEAD - expected = 'origin/master' - result = self._repo._current_ref_from_branch_command( - git_output) + self._repo._git_current_branch = self._git_current_branch(False, '') + self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') + self._repo._git_current_hash = self._git_current_hash(True, 'abc123') + expected = 'foo_tag' + result = self._repo._current_ref() self.assertEqual(result, expected) - def test_ref_tracking_branch_ahead_behind(self): # pylint: disable=C0103 - """Test that we correctly identify we are on a tracking branch that is - ahead or behind the remote branch. - - """ - git_output = GIT_BRANCH_OUTPUT_TRACKING_BRANCH_AHEAD_BEHIND - expected = 'origin/master' - result = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(result, expected) + def test_ref_detached_hash(self): + """Test that we can identify ref is detached at a hash - def test_ref_untracked_branch(self): - """Test that we correctly identify we are on an untracked branch """ - git_output = GIT_BRANCH_OUTPUT_UNTRACKED_BRANCH - expected = 'feature3' - result = self._repo._current_ref_from_branch_command( - git_output) + self._repo._git_current_branch = self._git_current_branch(False, '') + self._repo._git_current_tag = self._git_current_tag(False, '') + self._repo._git_current_hash = self._git_current_hash(True, 'abc123') + expected = 'abc123' + result = self._repo._current_ref() self.assertEqual(result, expected) def test_ref_none(self): - """Test that we can handle an empty string for output, e.g. not an git - repo. - + """Test that we correctly identify that we're not in a git repo. """ - git_output = EMPTY_STR - received = self._repo._current_ref_from_branch_command( - git_output) - self.assertEqual(received, EMPTY_STR) + self._repo._git_current_branch = self._git_current_branch(False, '') + self._repo._git_current_tag = self._git_current_tag(False, '') + self._repo._git_current_hash = self._git_current_hash(False, '') + result = self._repo._current_ref() + self.assertEqual(result, EMPTY_STR) class TestGitRepositoryCheckSync(unittest.TestCase): @@ -264,6 +198,12 @@ def setUp(self): model = ExternalsDescriptionDict(data) repo = model[self._name][ExternalsDescription.REPO] self._repo = GitRepository('test', repo) + # The unit tests here don't care about the result of + # _current_ref, but we replace it here so that we don't need to + # worry about calling a possibly slow and possibly + # error-producing command (since _current_ref calls various git + # functions): + self._repo._current_ref = self._current_ref_empty self._create_tmp_git_dir() def tearDown(self): @@ -287,52 +227,47 @@ def _remove_tmp_git_dir(self): # mock methods replacing git system calls # @staticmethod - def _git_branch_empty(): - """Return an empty info string. Simulates git info failing. + def _current_ref_empty(): + """Return an empty string. """ return EMPTY_STR @staticmethod - def _git_branch_detached_tag(): - """Return an info sting that is a checkouted tag - """ - return GIT_BRANCH_OUTPUT_DETACHED_TAG - - @staticmethod - def _git_branch_detached_hash(): + def _git_remote_origin_upstream(): """Return an info string that is a checkout hash """ - return GIT_BRANCH_OUTPUT_DETACHED_HASH + return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM @staticmethod - def _git_branch_detached_branch(): + def _git_remote_none(): """Return an info string that is a checkout hash """ - return GIT_BRANCH_OUTPUT_DETACHED_BRANCH + return EMPTY_STR @staticmethod - def _git_branch_untracked_branch(): - """Return an info string that is a checkout branch + def _git_current_hash(myhash): + """Return a function that takes the place of repo._git_current_hash, + which returns the given hash """ - return GIT_BRANCH_OUTPUT_UNTRACKED_BRANCH + def my_git_current_hash(): + """mock function that can take the place of repo._git_current_hash""" + return 0, myhash + return my_git_current_hash - @staticmethod - def _git_branch_tracked_branch(): - """Return an info string that is a checkout branch - """ - return GIT_BRANCH_OUTPUT_TRACKING_BRANCH + def _git_revparse_commit(self, expected_ref, mystatus, myhash): + """Return a function that takes the place of + repo._git_revparse_commit, which returns a tuple: + (mystatus, myhash). - @staticmethod - def _git_remote_origin_upstream(): - """Return an info string that is a checkout hash - """ - return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM + Expects the passed-in ref to equal expected_ref - @staticmethod - def _git_remote_none(): - """Return an info string that is a checkout hash + status = 0 implies success, non-zero implies failure """ - return EMPTY_STR + def my_git_revparse_commit(ref): + """mock function that can take the place of repo._git_revparse_commit""" + self.assertEqual(expected_ref, ref) + return mystatus, myhash + return my_git_revparse_commit # ---------------------------------------------------------------- # @@ -353,105 +288,71 @@ def test_sync_dir_not_exist(self): self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) def test_sync_dir_exist_no_git_info(self): - """Test that an empty info string returns an unknown status + """Test that a non-existent git repo returns an unknown status """ stat = ExternalStatus() - # Now we over-ride the _git_branch method on the repo to return + # Now we over-ride the _git_remote_verbose method on the repo to return # a known value without requiring access to git. self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._git_branch_vv = self._git_branch_empty + self._repo._tag = 'tag1' + self._repo._git_current_hash = self._git_current_hash('') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'tag1', 1, '') self._repo._check_sync(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.UNKNOWN) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - # ---------------------------------------------------------------- + # ------------------------------------------------------------------------ # - # Tests where external description specifies a tag - # - # Perturbations of working dir state: on detached - # {tag|branch|hash}, tracking branch, untracked branch. + # Tests where version in configuration file is not a valid reference # - # ---------------------------------------------------------------- - def test_sync_tag_on_detached_tag(self): - """Test expect tag on detached tag --> status ok - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' - self._repo._tag = 'tag1' - self._repo._git_branch_vv = self._git_branch_detached_tag - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_tag_on_diff_tag(self): - """Test expect tag on diff tag --> status modified - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' - self._repo._tag = 'tag2' - self._repo._git_branch_vv = self._git_branch_detached_tag - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_tag_on_detached_hash(self): - """Test expect tag on detached hash --> status modified - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' - self._repo._tag = 'tag1' - self._repo._git_branch_vv = self._git_branch_detached_hash - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_tag_on_detached_branch(self): - """Test expect tag on detached branch --> status modified + # ------------------------------------------------------------------------ + def test_sync_invalid_reference(self): + """Test that an invalid reference returns out-of-sync """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' self._repo._tag = 'tag1' - self._repo._git_branch_vv = self._git_branch_detached_branch + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'tag1', 1, '') self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_tag_on_tracking_branch(self): - """Test expect tag on tracking branch --> status modified + # ---------------------------------------------------------------- + # + # Tests where external description specifies a tag + # + # ---------------------------------------------------------------- + def test_sync_tag_on_same_hash(self): + """Test expect tag on same hash --> status ok """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' self._repo._tag = 'tag1' - self._repo._git_branch_vv = self._git_branch_tracked_branch + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'tag1', 0, 'abc123') self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) + self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_tag_on_untracked_branch(self): - """Test expect tag on untracked branch --> status modified + def test_sync_tag_on_different_hash(self): + """Test expect tag on a different hash --> status modified """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = '' self._repo._tag = 'tag1' - self._repo._git_branch_vv = self._git_branch_untracked_branch + self._repo._git_current_hash = self._git_current_hash('def456') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'tag1', 0, 'abc123') self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) # check_sync should only modify the sync_state, not clean_state @@ -459,115 +360,80 @@ def test_sync_tag_on_untracked_branch(self): # ---------------------------------------------------------------- # - # Tests where external description specifies a branch - # - # Perturbations of working dir state: on detached - # {tag|branch|hash}, tracking branch, untracked branch. + # Tests where external description specifies a hash # # ---------------------------------------------------------------- - def test_sync_branch_on_detached_branch_same_remote(self): - """Test expect branch on detached branch with same remote --> status ok + def test_sync_hash_on_same_hash(self): + """Test expect hash on same hash --> status ok """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = 'feature-2' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_detached_branch + self._repo._hash = 'abc' + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'abc', 0, 'abc123') self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_branch_on_detached_branch_diff_remote(self): - """Test expect branch on detached branch, different remote --> status modified - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = 'feature-2' - self._repo._tag = '' - self._repo._url = '/path/to/other/repo' - self._repo._git_branch_vv = self._git_branch_detached_branch - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_branch_on_detached_branch_diff_remote2(self): - """Test expect branch on detached branch, different remote --> status modified + def test_sync_hash_on_different_hash(self): + """Test expect hash on a different hash --> status modified """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = 'feature-2' - self._repo._tag = '' - self._repo._url = '/path/to/local/repo2' - self._repo._git_branch_vv = self._git_branch_detached_branch - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_branch_on_diff_branch(self): - """Test expect branch on diff branch --> status modified - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = 'nice_new_feature' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_detached_branch + self._repo._hash = 'abc' + self._repo._git_current_hash = self._git_current_hash('def456') + self._repo._git_revparse_commit = self._git_revparse_commit( + 'abc', 0, 'abc123') self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_branch_on_detached_hash(self): - """Test expect branch on detached hash --> status modified + # ---------------------------------------------------------------- + # + # Tests where external description specifies a branch + # + # ---------------------------------------------------------------- + def test_sync_branch_on_same_hash(self): + """Test expect branch on same hash --> status ok """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_detached_hash + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('origin/feature-2', 0, 'abc123')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) + self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_branch_on_detached_tag(self): - """Test expect branch on detached tag --> status modified + def test_sync_branch_on_diff_hash(self): + """Test expect branch on diff hash --> status modified """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_detached_tag + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('origin/feature-2', 0, 'def456')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - def test_sync_branch_on_tracking_branch_same_remote(self): - """Test expect branch on tracking branch with same remote --> status ok - - """ - stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream - self._repo._branch = 'feature-2' - self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_tracked_branch - self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) - - def test_sync_branch_on_tracking_branch_diff_remote(self): - """Test expect branch on tracking branch with different remote--> - status modified + def test_sync_branch_diff_remote(self): + """Test _determine_remote_name with a different remote """ stat = ExternalStatus() @@ -575,30 +441,28 @@ def test_sync_branch_on_tracking_branch_diff_remote(self): self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/other/repo' - self._repo._git_branch_vv = self._git_branch_tracked_branch + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('upstream/feature-2', 0, 'def456')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) + # The test passes if _git_revparse_commit is called with the + # expected argument - def test_sync_branch_on_untracked_branch(self): - """Test expect branch on untracked branch --> status modified - - NOTE(bja, 2017-11) the externals description url is always a - remote repository. A local untracked branch only exists - locally, therefore it is always a modified state, even if this - is what the user wants. + def test_sync_branch_diff_remote2(self): + """Test _determine_remote_name with a different remote """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_untracked_branch + self._repo._url = '/path/to/local/repo2' + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('other/feature-2', 0, 'def789')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) - self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) - # check_sync should only modify the sync_state, not clean_state - self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) + # The test passes if _git_revparse_commit is called with the + # expected argument def test_sync_branch_on_unknown_remote(self): """Test expect branch, but remote is unknown --> status modified @@ -609,7 +473,9 @@ def test_sync_branch_on_unknown_remote(self): self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/unknown/repo' - self._repo._git_branch_vv = self._git_branch_untracked_branch + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('unknown_remote/feature-2', 1, '')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.MODEL_MODIFIED) # check_sync should only modify the sync_state, not clean_state @@ -619,193 +485,25 @@ def test_sync_branch_on_untracked_local(self): """Test expect branch, on untracked branch in local repo --> status ok Setting the externals description to '.' indicates that the - user only want's to consider the current local repo state + user only wants to consider the current local repo state without fetching from remotes. This is required to preserve the current branch of a repository during an update. - NOTE(bja, 2017-11) the externals description is always a - remote repository. A local untracked branch only exists - locally, therefore it is always a modified state, even if this - is what the user wants. - """ stat = ExternalStatus() self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature3' self._repo._tag = '' - self._repo._git_branch_vv = self._git_branch_untracked_branch self._repo._url = '.' + self._repo._git_current_hash = self._git_current_hash('abc123') + self._repo._git_revparse_commit = ( + self._git_revparse_commit('feature3', 0, 'abc123')) self._repo._check_sync_logic(stat, self.TMP_FAKE_DIR) self.assertEqual(stat.sync_state, ExternalStatus.STATUS_OK) # check_sync should only modify the sync_state, not clean_state self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) -class TestGitRegExp(unittest.TestCase): - """Test that the regular expressions in the GitRepository class - capture intended strings - - """ - - def setUp(self): - """Common constans - """ - self._detached_git_v2_tmpl = string.Template( - '* (HEAD detached at $ref) 36418b4 Work on feature-2') - - self._detached_git_v1_tmpl = string.Template( - '* (detached from $ref) 36418b4 Work on feature-2') - - self._tracking_tmpl = string.Template( - '* feature-2 36418b4 [$ref] Work on feature-2') - - # - # RE_DETACHED - # - def test_re_detached_alphnum(self): - """Test re correctly matches alphnumeric (basic debugging) - """ - value = 'feature2' - input_str = self._detached_git_v2_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - input_str = self._detached_git_v1_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_detached_underscore(self): - """Test re matches with underscore - """ - value = 'feature_2' - input_str = self._detached_git_v2_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - input_str = self._detached_git_v1_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_detached_hyphen(self): - """Test re matches - - """ - value = 'feature-2' - input_str = self._detached_git_v2_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - input_str = self._detached_git_v1_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_detached_period(self): - """Test re matches . - """ - value = 'feature.2' - input_str = self._detached_git_v2_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - input_str = self._detached_git_v1_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_detached_slash(self): - """Test re matches / - """ - value = 'feature/2' - input_str = self._detached_git_v2_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - input_str = self._detached_git_v1_tmpl.substitute(ref=value) - match = GitRepository.RE_DETACHED.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - # - # RE_TRACKING - # - def test_re_tracking_alphnum(self): - """Test re matches alphanumeric for basic debugging - """ - value = 'feature2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_tracking_underscore(self): - """Test re matches _ - """ - value = 'feature_2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_tracking_hyphen(self): - """Test re matches - - """ - value = 'feature-2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_tracking_period(self): - """Test re match . - """ - value = 'feature.2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_tracking_slash(self): - """Test re matches / - """ - value = 'feature/2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), value) - - def test_re_tracking_colon(self): - """Test re rejects names with colons because they are invalid for git - tag and branch names - - """ - value = 'feature:2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNone(match) - - def test_re_tracking_ahead(self): - """Test re matches correctly with the ': ahead' syntax from git - """ - value = 'feature-2: ahead 3' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), 'feature-2') - - def test_re_tracking_ahead_behind(self): - """Test re matches correctly with the ': ahead 3, behind 2' syntax - from git - - """ - value = 'feature-2: ahead 3, behind 2' - input_str = self._tracking_tmpl.substitute(ref=value) - match = GitRepository.RE_TRACKING.search(input_str) - self.assertIsNotNone(match) - self.assertEqual(match.group(1), 'feature-2') - - class TestGitStatusPorcelain(unittest.TestCase): """Test parsing of output from git status --porcelain=v1 -z """ @@ -849,7 +547,8 @@ def setUp(self): ExternalsDescription.TAG: 'very_useful_tag', ExternalsDescription.BRANCH: EMPTY_STR, - ExternalsDescription.HASH: EMPTY_STR, } + ExternalsDescription.HASH: EMPTY_STR, + ExternalsDescription.SPARSE: EMPTY_STR, } self._repo = GitRepository('test', self._rdata) def test_remote_git_proto(self):