forked from ESCOMP/CAM-SIMA
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Squashed 'manage_externals/' changes from f1e9e99..14973c1
fde04e4 Merge pull request ESCOMP#138 from billsacks/add_python38_tests 37e4c4a Do not update dictionary in-place in loop 7e8474b Remove testing on mac os 7f41c56 Fix pylint issue 3065b0d Add travis-ci tests with python3.7 and python3.8 34fbf55 Add support for git sparse checkout 6c6ef9f Fix pylint errors 6a659ad Added test for sparse checkout and updated documentation 1443243 Support for git sparsecheckout via read-tree. a48558d Merge pull request ESCOMP#119 from gold2718/submodules f72ffe7 Do not try git submodule update if no .gitmodules file (git bug) 804e0af Fix a pylint error 45aef95 Addressed review concerns 7da5031 New capability to use git submodule information to checkout externals 1926530 Merge pull request ESCOMP#118 from mnlevy1981/svn_switch b1b028d Updates after testing 9ea73e6 Add --svn-ignore-ancestry argument fc5acda Merge pull request ESCOMP#114 from billsacks/fix_large_output_hang aa2eb71 Try getting travis-ci working on MacOS 96842b4 Fix pylint errors 813fe3c pylint: disable useless-object-inheritance c49d878 Rework execute_subprocess timeout handling 8fc0e5f Cleanup from 'make style' b0b23a6 Merge pull request ESCOMP#110 from gold2718/help_fix 3cbcd16 Fixed and clarified help documentation 025e6cb Merge pull request ESCOMP#107 from jedwards4b/ignore_empty_git_dir 489842b if you encounter an empty directory clone into it 0c5a2f6 Merge pull request ESCOMP#106 from billsacks/remove_logfile_message 7799e99 Remove message about checking the log file for more details 0427305 Merge pull request ESCOMP#103 from billsacks/no_logging 9bb46aa Make no-logging be the default 9af6b02 Merge pull request ESCOMP#102 from billsacks/explain_qmark 7f973ae Run through make style d077a57 Add message describing meaning of '?' 60fc03b Merge pull request ESCOMP#101 from ESMCI/catch_svn_error 28073ec add exception class 4fb7e47 catch errors from svn status --xml bfa4831 Merge pull request ESCOMP#98 from billsacks/quieter 7d12650 make style afb4f11 Make more git and svn commands quieter a465b4f add --quiet argument to improve performance b2f3ae8 Merge pull request ESCOMP#83 from jedwards4b/jedwards/components_arg 3f4c88f fix comment c1b5b09 remove unneeded logic 4fdf180 one more test f78d60f another test bf52ac6 add a test 91d4851 fix pylint issue 987df5a only use components if populated 98a810d add a components arg to checkout only select components 6923119 Merge pull request ESCOMP#90 from ESMCI/issue-86-detached-sync-status b11ad61 Merge branch 'master' into issue-86-detached-sync-status 3b624cf Merge pull request ESCOMP#93 from billsacks/work_on_coverage 2562830 Run a single coverage command rather than two separate commands d1de5f8 Return to starting directory after each test 144f7d9 Merge pull request ESCOMP#92 from billsacks/point_to_esmci 58b8d3e Point to location of repository 0b46d81 Point to correct location for build/coverage status a385070 fix pylint problems dcf17b6 make style cleanup 92d342c Rewrite _current_ref to use plumbing rather than parsing porcelain ca0a5d3 Rework some git repository functions, and major rework of unit tests 719383e Remove commented-out pdb.set_trace() call 376c780 Bugfix: detect and report 'detached from' correctly 21813e9 Add system test demonstrating failure to detect out of sync status. 1a7c59d Merge documentation update into master. 247fee1 Document return values of checkout.py: main git-subtree-dir: manage_externals git-subtree-split: 14973c1cadf0436638d012d128766af27e9a1090
- Loading branch information
Showing
25 changed files
with
1,807 additions
and
871 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,7 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| """ | ||
| Tool to assemble respositories represented in a model-description file. | ||
| Tool to assemble repositories represented in a model-description file. | ||
| If loaded as a module (e.g., in a component's buildcpp), it can be used | ||
| to check the validity of existing subdirectories and load missing sources. | ||
|
|
@@ -20,7 +20,7 @@ | |
| from manic.externals_description import read_externals_description_file | ||
| from manic.externals_status import check_safe_to_update_repos | ||
| from manic.sourcetree import SourceTree | ||
| from manic.utils import printlog | ||
| from manic.utils import printlog, fatal_error | ||
| from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME | ||
|
|
||
| if sys.hexversion < 0x02070000: | ||
|
|
@@ -48,17 +48,9 @@ def commandline_arguments(args=None): | |
| description = ''' | ||
| %(prog)s manages checking out groups of externals from revision | ||
| control based on a externals description file. By default only the | ||
| control based on an externals description file. By default only the | ||
| required externals are checkout out. | ||
| Operations performed by manage_externals utilities are explicit and | ||
| data driven. %(prog)s will always make the working copy *exactly* | ||
| match what is in the externals file when modifying the working copy of | ||
| a repository. | ||
| If %(prog)s isn't doing what you expected, double check the contents | ||
| of the externals description file. | ||
| Running %(prog)s without the '--status' option will always attempt to | ||
| synchronize the working copy to exactly match the externals description. | ||
| ''' | ||
|
|
@@ -76,7 +68,7 @@ def commandline_arguments(args=None): | |
| $ git clone [email protected]/{SOME_ORG}/some-project some-project-dev | ||
| and you need to checkout the sub-project externals, then the root of the | ||
| source tree is /path/to/some-project-dev. Do *NOT* run %(prog)s | ||
| source tree remains /path/to/some-project-dev. Do *NOT* run %(prog)s | ||
| from within /path/to/some-project-dev/sub-project | ||
| The root of the source tree will be referred to as `${SRC_ROOT}` below. | ||
|
|
@@ -109,7 +101,7 @@ def commandline_arguments(args=None): | |
| description file: | ||
| $ cd ${SRC_ROOT} | ||
| $ ./manage_externals/%(prog)s --excernals my-externals.cfg | ||
| $ ./manage_externals/%(prog)s --externals my-externals.cfg | ||
| * Status summary of the repositories managed by %(prog)s: | ||
|
|
@@ -178,8 +170,9 @@ def commandline_arguments(args=None): | |
| Note: 'externals_only' will only process the external's own | ||
| external description file without trying to manage a repository | ||
| for the component. This is used for retreiving externals for | ||
| standalone components like cam and clm. If the source root of the | ||
| for the component. This is used for retrieving externals for | ||
| standalone components like cam and ctsm which also serve as | ||
| sub-components within a larger project. If the source root of the | ||
| externals_only component is the same as the main source root, then | ||
| the local path must be set to '.', the unix current working | ||
| directory, e. g. 'local_path = .' | ||
|
|
@@ -219,15 +212,54 @@ def commandline_arguments(args=None): | |
| * externals (string) : used to make manage_externals aware of | ||
| sub-externals required by an external. This is a relative path to | ||
| the external's root directory. For example, the main externals | ||
| description has an external checkout out at 'src/useful_library'. | ||
| useful_library requires additional externals to be complete. | ||
| Those additional externals are managed from the source root by the | ||
| externals description file pointed 'useful_library/sub-xternals.cfg', | ||
| Then the main 'externals' field in the top level repo should point to | ||
| 'sub-externals.cfg'. | ||
| the external's root directory. For example, if LIBX is often used | ||
| as a sub-external, it might have an externals file (for its | ||
| externals) called Externals_LIBX.cfg. To use libx as a standalone | ||
| checkout, it would have another file, Externals.cfg with the | ||
| following entry: | ||
| [ libx ] | ||
| local_path = . | ||
| protocol = externals_only | ||
| externals = Externals_LIBX.cfg | ||
| required = True | ||
| Now, %(prog)s will process Externals.cfg and also process | ||
| Externals_LIBX.cfg as if it was a sub-external. | ||
| * from_submodule (True / False) : used to pull the repo_url, local_path, | ||
| and hash properties for this external from the .gitmodules file in | ||
| this repository. Note that the section name (the entry in square | ||
| brackets) must match the name in the .gitmodules file. | ||
| If from_submodule is True, the protocol must be git and no repo_url, | ||
| local_path, hash, branch, or tag entries are allowed. | ||
| Default: False | ||
| * sparse (string) : used to control a sparse checkout. This optional | ||
| entry should point to a filename (path relative to local_path) that | ||
| contains instructions on which repository paths to include (or | ||
| exclude) from the working tree. | ||
| See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree | ||
| Default: sparse checkout is disabled | ||
| * Lines beginning with '#' or ';' are comments and will be ignored. | ||
| # Obtaining this tool, reporting issues, etc. | ||
| The master repository for manage_externals is | ||
| https://github.com/ESMCI/manage_externals. Any issues with this tool | ||
| should be reported there. | ||
| # Troubleshooting | ||
| * Lines begining with '#' or ';' are comments and will be ignored. | ||
| Operations performed by manage_externals utilities are explicit and | ||
| data driven. %(prog)s will always attempt to make the working copy | ||
| *exactly* match what is in the externals file when modifying the | ||
| working copy of a repository. | ||
| If %(prog)s is not doing what you expected, double check the contents | ||
| of the externals description file or examine the output of | ||
| ./manage_externals/%(prog)s --status | ||
| ''' | ||
|
|
||
|
|
@@ -238,6 +270,10 @@ def commandline_arguments(args=None): | |
| # | ||
| # user options | ||
| # | ||
| parser.add_argument("components", nargs="*", | ||
| help="Specific component(s) to checkout. By default, " | ||
| "all required externals are checked out.") | ||
|
|
||
| parser.add_argument('-e', '--externals', nargs='?', | ||
| default='Externals.cfg', | ||
| help='The externals description filename. ' | ||
|
|
@@ -249,16 +285,23 @@ def commandline_arguments(args=None): | |
| 'optional externals.') | ||
|
|
||
| parser.add_argument('-S', '--status', action='store_true', default=False, | ||
| help='Output status of the repositories managed by ' | ||
| help='Output the status of the repositories managed by ' | ||
| '%(prog)s. By default only summary information ' | ||
| 'is provided. Use verbose output to see details.') | ||
| 'is provided. Use the verbose option to see details.') | ||
|
|
||
| parser.add_argument('-v', '--verbose', action='count', default=0, | ||
| help='Output additional information to ' | ||
| 'the screen and log file. This flag can be ' | ||
| 'used up to two times, increasing the ' | ||
| 'verbosity level each time.') | ||
|
|
||
| parser.add_argument('--svn-ignore-ancestry', action='store_true', default=False, | ||
| help='By default, subversion will abort if a component is ' | ||
| 'already checked out and there is no common ancestry with ' | ||
| 'the new URL. This flag passes the "--ignore-ancestry" flag ' | ||
| 'to the svn switch call. (This is not recommended unless ' | ||
| 'you are sure about what you are doing.)') | ||
|
|
||
| # | ||
| # developer options | ||
| # | ||
|
|
@@ -270,8 +313,15 @@ def commandline_arguments(args=None): | |
| help='DEVELOPER: output additional debugging ' | ||
| 'information to the screen and log file.') | ||
|
|
||
| parser.add_argument('--no-logging', action='store_true', | ||
| help='DEVELOPER: disable logging.') | ||
| logging_group = parser.add_mutually_exclusive_group() | ||
|
|
||
| logging_group.add_argument('--logging', dest='do_logging', | ||
| action='store_true', | ||
| help='DEVELOPER: enable logging.') | ||
| logging_group.add_argument('--no-logging', dest='do_logging', | ||
| action='store_false', default=False, | ||
| help='DEVELOPER: disable logging ' | ||
| '(this is the default)') | ||
|
|
||
| if args: | ||
| options = parser.parse_args(args) | ||
|
|
@@ -290,8 +340,13 @@ def main(args): | |
| Function to call when module is called from the command line. | ||
| Parse externals file and load required repositories or all repositories if | ||
| the --all option is passed. | ||
| Returns a tuple (overall_status, tree_status). overall_status is 0 | ||
| on success, non-zero on failure. tree_status gives the full status | ||
| *before* executing the checkout command - i.e., the status that it | ||
| used to determine if it's safe to proceed with the checkout. | ||
| """ | ||
| if not args.no_logging: | ||
| if args.do_logging: | ||
| logging.basicConfig(filename=LOG_FILE_NAME, | ||
| format='%(levelname)s : %(asctime)s : %(message)s', | ||
| datefmt='%Y-%m-%d %H:%M:%S', | ||
|
|
@@ -306,9 +361,16 @@ def main(args): | |
|
|
||
| root_dir = os.path.abspath(os.getcwd()) | ||
| external_data = read_externals_description_file(root_dir, args.externals) | ||
| external = create_externals_description(external_data) | ||
| external = create_externals_description( | ||
| external_data, components=args.components) | ||
|
|
||
| source_tree = SourceTree(root_dir, external) | ||
| for comp in args.components: | ||
| if comp not in external.keys(): | ||
| fatal_error( | ||
| "No component {} found in {}".format( | ||
| comp, args.externals)) | ||
|
|
||
| source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry) | ||
| printlog('Checking status of externals: ', end='') | ||
| tree_status = source_tree.status() | ||
| printlog('') | ||
|
|
@@ -338,13 +400,23 @@ def main(args): | |
| (2) Alternatively, you do not have to rely on {program_name}. Instead, you | ||
| can manually update out-of-sync externals (labeled with 's' above) | ||
| as described in the configuration file {config_file}. | ||
| The external repositories labeled with '?' above are not under version | ||
| control using the expected protocol. If you are sure you want to switch | ||
| protocols, and you don't have any work you need to save from this | ||
| directory, then run "rm -rf [directory]" before re-running the | ||
| checkout_externals tool. | ||
| """.format(program_name=program_name, config_file=args.externals) | ||
|
|
||
| printlog('-' * 70) | ||
| printlog(msg) | ||
| printlog('-' * 70) | ||
| else: | ||
| source_tree.checkout(args.verbose, load_all) | ||
| if not args.components: | ||
| source_tree.checkout(args.verbose, load_all) | ||
| for comp in args.components: | ||
| source_tree.checkout(args.verbose, load_all, load_comp=comp) | ||
| printlog('') | ||
|
|
||
| logging.info('%s completed without exceptions.', program_name) | ||
|
|
||
Oops, something went wrong.