forked from ansible-community/antsibull-build
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This adds a new validate-tags command to retrieve the tags in each collection's git repository and ensure that the current version is tagged. Currently, this is a separate command, but I'd like to have `antsibull-build` include the data from `get_collections_tags()` in ansible-build-data and the ansible sdist after this has gotten more testing/feedback. I also split CollectionsMetadata into a separate file and added the new keys. Previously, this class was part of changelog.py and only used for retrieving changelog URLs. Relates: ansible-community/community-topics#148 Depends-on: ansible-community/ansible-build-data#176
- Loading branch information
Showing
5 changed files
with
226 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| # Author: Felix Fontein <[email protected]> | ||
| # Author: Toshio Kuratomi <[email protected]> | ||
| # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or | ||
| # https://www.gnu.org/licenses/gpl-3.0.txt) | ||
| # SPDX-License-Identifier: GPL-3.0-or-later | ||
| # SPDX-FileCopyrightText: Ansible Project, 2020 | ||
|
|
||
| """ | ||
| Classes to encapsulate collection metadata from collection-meta.yaml | ||
| """ | ||
| import typing as t | ||
| import os | ||
|
|
||
| from antsibull_core.yaml import load_yaml_file | ||
|
|
||
|
|
||
| class CollectionMetadata: | ||
| ''' | ||
| Stores metadata about one collection. | ||
| ''' | ||
|
|
||
| changelog_url: t.Optional[str] | ||
|
|
||
| def __init__(self, source: t.Optional[t.Mapping[str, t.Any]] = None): | ||
| if source is None: | ||
| source = {} | ||
| self.changelog_url: t.Optional[str] = source.get('changelog-url') | ||
| self.collection_directory: t.Optional[str] = source.get('collection-directory') | ||
| self.repository: t.Optional[str] = source.get('repository') | ||
|
|
||
|
|
||
| class CollectionsMetadata: | ||
| ''' | ||
| Stores metadata about a set of collections. | ||
| ''' | ||
|
|
||
| data: t.Dict[str, CollectionMetadata] | ||
|
|
||
| def __init__(self, deps_dir: t.Optional[str]): | ||
| self.data = {} | ||
| if deps_dir is not None: | ||
| collection_meta_path = os.path.join(deps_dir, 'collection-meta.yaml') | ||
| if os.path.exists(collection_meta_path): | ||
| data = load_yaml_file(collection_meta_path) | ||
| if data and 'collections' in data: | ||
| for collection_name, collection_data in data['collections'].items(): | ||
| self.data[collection_name] = CollectionMetadata(collection_data) | ||
|
|
||
| def get_meta(self, collection_name: str) -> CollectionMetadata: | ||
| result = self.data.get(collection_name) | ||
| if result is None: | ||
| result = CollectionMetadata() | ||
| self.data[collection_name] = result | ||
| return result |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| # GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or | ||
| # https://www.gnu.org/licenses/gpl-3.0.txt) | ||
| # SPDX-License-Identifier: GPL-3.0-or-later | ||
| # SPDX-FileCopyrightText: 2022 Maxwell G <[email protected]> | ||
|
|
||
| """ | ||
| Validate that collections tag their releases in their respective git repositories | ||
| """ | ||
| import asyncio | ||
| import os | ||
| import re | ||
| import typing as t | ||
|
|
||
| import asyncio_pool # type: ignore[import] | ||
|
|
||
| from antsibull_core.logging import log | ||
|
|
||
| from antsibull_core import app_context | ||
| from antsibull_core.dependency_files import DepsFile | ||
| from antsibull_core.yaml import store_yaml_file, load_yaml_file | ||
|
|
||
| from antsibull.collection_meta import CollectionsMetadata | ||
|
|
||
| mlog = log.fields(mod=__name__) | ||
|
|
||
|
|
||
| def validate_tags_command() -> int: | ||
| app_ctx = app_context.app_ctx.get() | ||
| if app_ctx.extra['input']: | ||
| tag_data = load_yaml_file(app_ctx.extra['input']) | ||
| else: | ||
| tag_data = asyncio.run(get_collections_tags()) | ||
| if app_ctx.extra['output']: | ||
| store_yaml_file(app_ctx.extra['output'], tag_data) | ||
| return validate_tags(tag_data) | ||
|
|
||
|
|
||
| def validate_tags(tag_data: t.Dict[str, t.Dict[str, t.Optional[str]]]) -> int: | ||
| r = 0 | ||
| flog = mlog.fields(func='validate_tags') | ||
| for name, data in tag_data.items(): | ||
| if not data['repository']: | ||
| flog.error( | ||
| f"{name}'s repository is not specified at all in collection-meta.yaml" | ||
| ) | ||
| r = 1 | ||
| continue | ||
| if not data['tag']: | ||
| flog.error( | ||
| f"{name} {data['version']} is not tagged in " | ||
| f"{data['repository']}" | ||
| ) | ||
| r = 1 | ||
| return r | ||
|
|
||
|
|
||
| async def get_collections_tags() -> t.Dict[str, t.Dict[str, t.Optional[str]]]: | ||
| app_ctx = app_context.app_ctx.get() | ||
| lib_ctx = app_context.lib_ctx.get() | ||
|
|
||
| deps_filename = os.path.join( | ||
| app_ctx.extra['data_dir'], app_ctx.extra['deps_file'] | ||
| ) | ||
| deps_data = DepsFile(deps_filename).parse() | ||
| meta_data = CollectionsMetadata(app_ctx.extra['data_dir']) | ||
|
|
||
| async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: | ||
| collection_tags = {} | ||
| for name, data in meta_data.data.items(): | ||
| collection_tags[name] = pool.spawn_n( | ||
| _get_collection_tags(deps_data.deps[name], data) | ||
| ) | ||
| collection_tags = { | ||
| name: await data for name, data in collection_tags.items() | ||
| } | ||
| return collection_tags | ||
|
|
||
|
|
||
| async def _get_collection_tags( | ||
| version: str, meta_data=t.Dict[str, t.Optional[str]] | ||
| ) -> t.Dict[str, t.Optional[str]]: | ||
| flog = mlog.fields(func='_get_collection_tags') | ||
| repository = meta_data.repository | ||
| data: t.Dict[str, t.Optional[str]] = dict( | ||
| version=version, repository=repository, tag=None | ||
| ) | ||
| if meta_data.collection_directory: | ||
| data['collection_directory'] = meta_data.collection_directory | ||
| if not repository: | ||
| flog.debug("'repository' is None. Exitting...") | ||
| return data | ||
| async for tag in _get_tags(repository): | ||
| if _normalize_tag(tag) == version: | ||
| data['tag'] = tag | ||
| break | ||
| return data | ||
|
|
||
|
|
||
| async def _get_tags(repository) -> t.AsyncGenerator[str, None]: | ||
| flog = mlog.fields(func='_get_tags') | ||
| args = ( | ||
| 'git', | ||
| 'ls-remote', | ||
| '--refs', | ||
| '--tags', | ||
| repository, | ||
| ) | ||
| flog.debug(f'Running {args}') | ||
| proc = await asyncio.create_subprocess_exec( | ||
| *args, | ||
| stdout=asyncio.subprocess.PIPE, | ||
| stderr=asyncio.subprocess.PIPE, | ||
| ) | ||
| stdout, stderr = await proc.communicate() | ||
| flog.fields(stderr=stderr).debug('Ran git ls-remote') | ||
| if proc.returncode != 0: | ||
| flog.error(f'Failed to fetch tags for {repository}') | ||
| return | ||
| tags: t.List[str] = stdout.decode('utf-8').splitlines() | ||
| matcher: t.Pattern[str] = re.compile(r'^.*refs/tags/(.*)$') | ||
| if not tags: | ||
| flog.warning(f'{repository} does not have any tags') | ||
| return | ||
| for tag in tags: | ||
| match = matcher.match(tag) | ||
| if match: | ||
| yield match.group(1) | ||
| else: | ||
| flog.debug(f'git ls-remote output line skipped: {tag}') | ||
|
|
||
|
|
||
| def _normalize_tag(tag: str) -> str: | ||
| if tag.startswith('v'): | ||
| tag = tag[1:] | ||
| return tag |