From 8a043078f5422221bf801ca4e2bd15756987ad29 Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Thu, 10 Nov 2022 12:22:59 -0600 Subject: [PATCH] Add validate-tags command This adds a new validate-tags command to retrieve the tags in each collection's git repository and ensure that the current version is tagged. Currently, this is a separate command, but I'd like to have `antsibull-build` include the data from `get_collections_tags()` in ansible-build-data and the ansible sdist after this has gotten more testing/feedback. I also split CollectionsMetadata into a separate file and added the new keys. Previously, this class was part of changelog.py and only used for retrieving changelog URLs. Relates: https://github.com/ansible-community/community-topics/issues/148 Depends-on: https://github.com/ansible-community/ansible-build-data/pull/176 --- src/antsibull/build_changelog.py | 3 +- src/antsibull/changelog.py | 42 +-------- src/antsibull/cli/antsibull_build.py | 33 ++++++- src/antsibull/collection_meta.py | 54 +++++++++++ src/antsibull/tagging.py | 135 +++++++++++++++++++++++++++ 5 files changed, 226 insertions(+), 41 deletions(-) create mode 100644 src/antsibull/collection_meta.py create mode 100644 src/antsibull/tagging.py diff --git a/src/antsibull/build_changelog.py b/src/antsibull/build_changelog.py index e38f957c0..52b6fb2f8 100644 --- a/src/antsibull/build_changelog.py +++ b/src/antsibull/build_changelog.py @@ -21,7 +21,8 @@ from antsibull_core import app_context -from .changelog import Changelog, ChangelogData, ChangelogEntry, CollectionsMetadata, get_changelog +from .changelog import Changelog, ChangelogData, ChangelogEntry, get_changelog +from .collection_meta import CollectionsMetadata # diff --git a/src/antsibull/changelog.py b/src/antsibull/changelog.py index 4e9ac5352..31a718896 100644 --- a/src/antsibull/changelog.py +++ b/src/antsibull/changelog.py @@ -30,7 +30,9 @@ from antsibull_core.ansible_core import get_ansible_core from antsibull_core.dependency_files import DepsFile, DependencyFileData from antsibull_core.galaxy import CollectionDownloader -from antsibull_core.yaml import load_yaml_bytes, load_yaml_file +from antsibull_core.yaml import load_yaml_bytes + +from antsibull.collection_meta import CollectionsMetadata class ChangelogData: @@ -383,44 +385,6 @@ def __init__(self, version: PypiVer, version_str: str, collector, collection_version, prev_collection_version, added)) -class CollectionMetadata: - ''' - Stores metadata about one collection. - ''' - - changelog_url: t.Optional[str] - - def __init__(self, source: t.Optional[t.Mapping[str, t.Any]] = None): - if source is None: - source = {} - self.changelog_url = source.get('changelog-url') - - -class CollectionsMetadata: - ''' - Stores metadata about a set of collections. - ''' - - data: t.Dict[str, CollectionMetadata] - - def __init__(self, deps_dir: t.Optional[str]): - self.data = {} - if deps_dir is not None: - collection_meta_path = os.path.join(deps_dir, 'collection-meta.yaml') - if os.path.exists(collection_meta_path): - data = load_yaml_file(collection_meta_path) - if data and 'collections' in data: - for collection_name, collection_data in data['collections'].items(): - self.data[collection_name] = CollectionMetadata(collection_data) - - def get_meta(self, collection_name: str) -> CollectionMetadata: - result = self.data.get(collection_name) - if result is None: - result = CollectionMetadata() - self.data[collection_name] = result - return result - - class Changelog: ansible_version: PypiVer ansible_ancestor_version: t.Optional[PypiVer] diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index c81726a09..8f6799fb2 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -32,6 +32,7 @@ from ..build_changelog import build_changelog # noqa: E402 from ..dep_closure import validate_dependencies_command # noqa: E402 from ..new_ansible import new_ansible_command # noqa: E402 +from ..tagging import validate_tags_command # noqa: E402 # pylint: enable=wrong-import-position @@ -48,6 +49,7 @@ 'changelog': build_changelog, 'rebuild-single': rebuild_single_command, 'validate-deps': validate_dependencies_command, + 'validate-tags': validate_tags_command, # Old names, deprecated 'new-acd': new_ansible_command, 'build-single': build_single_command, @@ -87,7 +89,7 @@ def _normalize_commands(args: argparse.Namespace) -> None: def _normalize_build_options(args: argparse.Namespace) -> None: - if args.command in ('validate-deps', ): + if args.command in ('validate-deps', 'validate-tags'): return if not os.path.isdir(args.data_dir): @@ -199,6 +201,15 @@ def _normalize_collection_build_options(args: argparse.Namespace) -> None: raise InvalidArgumentError(f'{args.collection_dir} must be an existing directory') +def _normalize_validate_tags_options(args: argparse.Namespace) -> None: + if args.command not in ('validate-deps',): + return + if args.deps_file is None: + args.deps_file = DEFAULT_FILE_BASE + f'-{args.ansible_version}.deps' + if args.input and not os.path.exists(args.input): + raise InvalidArgumentError(f"{args.dep_file} does not exist!") + + def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: """ Parse and coerce the command line arguments. @@ -345,6 +356,25 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: help='Path to a ansible_collections directory containing a' ' collection tree to check.') + validate_tags = subparsers.add_parser('validate-tags', + parents=[build_parser]) + validate_tags.add_argument( + '--deps-file', + default=None, + help='File which contains the list of collections and' + ' versions which were included in this version of Ansible.' + ' This is considered to be relative to --data-dir.' + f' The default is {DEFAULT_FILE_BASE}-X.Y.Z.deps', + ) + tag_file = validate_tags.add_mutually_exclusive_group() + tag_file.add_argument('-i', + '--input', + help=('Collection tag data file to validate.' + 'Mutually exclusive with --output.')) + tag_file.add_argument('-o', + '--output', + help='Path to output a collection tag data file.' + 'If this is ommited, no tag data will be written') # Backwards compat subparsers.add_parser('new-acd', add_help=False, parents=[new_parser]) subparsers.add_parser('build-single', add_help=False, parents=[build_single_parser]) @@ -362,6 +392,7 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: _normalize_release_build_options(parsed_args) _normalize_release_rebuild_options(parsed_args) _normalize_collection_build_options(parsed_args) + _normalize_validate_tags_options(parsed_args) return parsed_args diff --git a/src/antsibull/collection_meta.py b/src/antsibull/collection_meta.py new file mode 100644 index 000000000..d497b6df3 --- /dev/null +++ b/src/antsibull/collection_meta.py @@ -0,0 +1,54 @@ +# Author: Felix Fontein +# Author: Toshio Kuratomi +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: Ansible Project, 2020 + +""" +Classes to encapsulate collection metadata from collection-meta.yaml +""" +import typing as t +import os + +from antsibull_core.yaml import load_yaml_file + + +class CollectionMetadata: + ''' + Stores metadata about one collection. + ''' + + changelog_url: t.Optional[str] + + def __init__(self, source: t.Optional[t.Mapping[str, t.Any]] = None): + if source is None: + source = {} + self.changelog_url: t.Optional[str] = source.get('changelog-url') + self.collection_directory: t.Optional[str] = source.get('collection-directory') + self.repository: t.Optional[str] = source.get('repository') + + +class CollectionsMetadata: + ''' + Stores metadata about a set of collections. + ''' + + data: t.Dict[str, CollectionMetadata] + + def __init__(self, deps_dir: t.Optional[str]): + self.data = {} + if deps_dir is not None: + collection_meta_path = os.path.join(deps_dir, 'collection-meta.yaml') + if os.path.exists(collection_meta_path): + data = load_yaml_file(collection_meta_path) + if data and 'collections' in data: + for collection_name, collection_data in data['collections'].items(): + self.data[collection_name] = CollectionMetadata(collection_data) + + def get_meta(self, collection_name: str) -> CollectionMetadata: + result = self.data.get(collection_name) + if result is None: + result = CollectionMetadata() + self.data[collection_name] = result + return result diff --git a/src/antsibull/tagging.py b/src/antsibull/tagging.py new file mode 100644 index 000000000..a81ed68ee --- /dev/null +++ b/src/antsibull/tagging.py @@ -0,0 +1,135 @@ +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: 2022 Maxwell G + +""" +Validate that collections tag their releases in their respective git repositories +""" +import asyncio +import os +import re +import typing as t + +import asyncio_pool # type: ignore[import] + +from antsibull_core.logging import log + +from antsibull_core import app_context +from antsibull_core.dependency_files import DepsFile +from antsibull_core.yaml import store_yaml_file, load_yaml_file + +from antsibull.collection_meta import CollectionsMetadata + +mlog = log.fields(mod=__name__) + + +def validate_tags_command() -> int: + app_ctx = app_context.app_ctx.get() + if app_ctx.extra['input']: + tag_data = load_yaml_file(app_ctx.extra['input']) + else: + tag_data = asyncio.run(get_collections_tags()) + if app_ctx.extra['output']: + store_yaml_file(app_ctx.extra['output'], tag_data) + return validate_tags(tag_data) + + +def validate_tags(tag_data: t.Dict[str, t.Dict[str, t.Optional[str]]]) -> int: + r = 0 + flog = mlog.fields(func='validate_tags') + for name, data in tag_data.items(): + if not data['repository']: + flog.error( + f"{name}'s repository is not specified at all in collection-meta.yaml" + ) + r = 1 + continue + if not data['tag']: + flog.error( + f"{name} {data['version']} is not tagged in " + f"{data['repository']}" + ) + r = 1 + return r + + +async def get_collections_tags() -> t.Dict[str, t.Dict[str, t.Optional[str]]]: + app_ctx = app_context.app_ctx.get() + lib_ctx = app_context.lib_ctx.get() + + deps_filename = os.path.join( + app_ctx.extra['data_dir'], app_ctx.extra['deps_file'] + ) + deps_data = DepsFile(deps_filename).parse() + meta_data = CollectionsMetadata(app_ctx.extra['data_dir']) + + async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: + collection_tags = {} + for name, data in meta_data.data.items(): + collection_tags[name] = pool.spawn_n( + _get_collection_tags(deps_data.deps[name], data) + ) + collection_tags = { + name: await data for name, data in collection_tags.items() + } + return collection_tags + + +async def _get_collection_tags( + version: str, meta_data=t.Dict[str, t.Optional[str]] +) -> t.Dict[str, t.Optional[str]]: + flog = mlog.fields(func='_get_collection_tags') + repository = meta_data.repository + data: t.Dict[str, t.Optional[str]] = dict( + version=version, repository=repository, tag=None + ) + if meta_data.collection_directory: + data['collection_directory'] = meta_data.collection_directory + if not repository: + flog.debug("'repository' is None. Exitting...") + return data + async for tag in _get_tags(repository): + if _normalize_tag(tag) == version: + data['tag'] = tag + break + return data + + +async def _get_tags(repository) -> t.AsyncGenerator[str, None]: + flog = mlog.fields(func='_get_tags') + args = ( + 'git', + 'ls-remote', + '--refs', + '--tags', + repository, + ) + flog.debug(f'Running {args}') + proc = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + flog.fields(stderr=stderr).debug('Ran git ls-remote') + if proc.returncode != 0: + flog.error(f'Failed to fetch tags for {repository}') + return + tags: t.List[str] = stdout.decode('utf-8').splitlines() + matcher: t.Pattern[str] = re.compile(r'^.*refs/tags/(.*)$') + if not tags: + flog.warning(f'{repository} does not have any tags') + return + for tag in tags: + match = matcher.match(tag) + if match: + yield match.group(1) + else: + flog.debug(f'git ls-remote output line skipped: {tag}') + + +def _normalize_tag(tag: str) -> str: + if tag.startswith('v'): + tag = tag[1:] + return tag