From 92d096cbc9707184b8559926530f3c6ef0d02f45 Mon Sep 17 00:00:00 2001 From: git-hyagi <45576767+git-hyagi@users.noreply.github.com> Date: Tue, 25 Jun 2024 16:54:03 -0300 Subject: [PATCH] Add repository_version param as a building context closes: #479 --- CHANGES/479.feature | 1 + .../app/global_access_conditions.py | 18 ++- pulp_container/app/serializers.py | 15 +++ pulp_container/app/tasks/builder.py | 23 +++- pulp_container/app/viewsets.py | 13 +- .../tests/functional/api/test_build_images.py | 127 +++++++++++++++++- staging_docs/admin/guides/build-image.md | 18 ++- 7 files changed, 202 insertions(+), 13 deletions(-) create mode 100644 CHANGES/479.feature diff --git a/CHANGES/479.feature b/CHANGES/479.feature new file mode 100644 index 000000000..612867f99 --- /dev/null +++ b/CHANGES/479.feature @@ -0,0 +1 @@ +Added support for using the file `repository_version` as the build context for a Containerfile. diff --git a/pulp_container/app/global_access_conditions.py b/pulp_container/app/global_access_conditions.py index abca8c772..9c1e7a042 100644 --- a/pulp_container/app/global_access_conditions.py +++ b/pulp_container/app/global_access_conditions.py @@ -1,10 +1,12 @@ from logging import getLogger -from pulpcore.plugin.models import Repository +from pulpcore.plugin.models import Repository, RepositoryVersion +from pulpcore.plugin.util import get_objects_for_user from pulpcore.plugin.viewsets import RepositoryVersionViewSet from pulp_container.app import models from pulp_container.app.viewsets import ContainerDistributionViewSet +from pulp_container.app import serializers _logger = getLogger(__name__) @@ -118,3 +120,17 @@ def has_distribution_perms(request, view, action, permission): return any( (request.user.has_perm(permission, distribution.cast()) for distribution in distributions) ) + + +def has_file_repo_perms(request, view, action, permission): + serializer = serializers.OCIBuildImageSerializer( + data=request.data, context={"request": request} + ) + serializer.is_valid(raise_exception=True) + repo_version = serializer.validated_data.get("repo_version", None) + if not repo_version: + return True + + repo_version_qs = RepositoryVersion.objects.filter(pk=repo_version) + file_repositories = get_objects_for_user(request.user, permission, repo_version_qs) + return repo_version_qs == file_repositories diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 3cac522bd..b01158336 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -756,6 +756,11 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer): "relative path (name) inside the /pulp_working_directory of the build container " "executing the Containerfile.", ) + repo_version = RepositoryVersionRelatedField( + required=False, + help_text=_("RepositoryVersion to be used as the build context for container images."), + allow_null=True, + ) def __init__(self, *args, **kwargs): """Initializer for OCIBuildImageSerializer.""" @@ -778,6 +783,12 @@ def validate(self, data): raise serializers.ValidationError( _("'containerfile' or 'containerfile_artifact' must " "be specified.") ) + + if not (("artifacts" in data) ^ ("repo_version" in data)): + raise serializers.ValidationError( + _("Only one of 'artifacts' or 'repo_version' should be provided!") + ) + artifacts = {} if "artifacts" in data: for url, relative_path in data["artifacts"].items(): @@ -800,6 +811,9 @@ def validate(self, data): e.detail[0] = "%s %s" % (e.detail[0], url) raise e data["artifacts"] = artifacts + if "repo_version" in data: + data["repo_version"] = data["repo_version"].pk + return data class Meta: @@ -809,6 +823,7 @@ class Meta: "repository", "tag", "artifacts", + "repo_version", ) diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 58daa3796..5eaf23cbd 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -1,3 +1,5 @@ +from gettext import gettext as _ + import json import os import shutil @@ -14,7 +16,10 @@ ) from pulp_container.constants import MEDIA_TYPE from pulp_container.app.utils import calculate_digest -from pulpcore.plugin.models import Artifact, ContentArtifact, Content +from pulpcore.plugin.models import Artifact, ContentArtifact, Content, RepositoryVersion +from pulp_file.app.models import FileContent + +from rest_framework.serializers import ValidationError def get_or_create_blob(layer_json, manifest, path): @@ -96,7 +101,7 @@ def add_image_from_directory_to_repository(path, repository, tag): def build_image_from_containerfile( - containerfile_pk=None, artifacts=None, repository_pk=None, tag=None + containerfile_pk=None, artifacts=None, repository_pk=None, tag=None, repo_version=None ): """ Builds an OCI container image from a Containerfile. @@ -111,6 +116,8 @@ def build_image_from_containerfile( container executing the Containerfile. repository_pk (str): The pk of a Repository to add the OCI container image tag (str): Tag name for the new image in the repository + repo_version: The pk of a RepositoryVersion with the artifacts used in the build context + of the Containerfile. Returns: A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container @@ -124,6 +131,18 @@ def build_image_from_containerfile( working_directory = os.path.abspath(working_directory) context_path = os.path.join(working_directory, "context") os.makedirs(context_path, exist_ok=True) + + if repo_version: + artifacts = {} + repo_version_artifacts = RepositoryVersion.objects.get(pk=repo_version).artifacts + files = FileContent.objects.filter( + digest__in=repo_version_artifacts.values("sha256") + ).values("_artifacts__pk", "relative_path") + if len(files) == 0: + raise ValidationError(_("No file found for the specified repository version.")) + for file in files: + artifacts[str(file["_artifacts__pk"])] = file["relative_path"] + for key, val in artifacts.items(): artifact = Artifact.objects.get(pk=key) dest_path = os.path.join(context_path, val) diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index e42be79f9..ba581dc18 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -5,6 +5,8 @@ http://docs.pulpproject.org/plugins/plugin-writer/index.html """ +from gettext import gettext as _ + import logging from django.db import IntegrityError @@ -15,6 +17,7 @@ from rest_framework import mixins from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied from pulpcore.plugin.models import RepositoryVersion from pulpcore.plugin.serializers import AsyncOperationResponseSerializer @@ -696,6 +699,7 @@ class ContainerRepositoryViewSet( "condition": [ "has_model_or_obj_perms:container.build_image_containerrepository", "has_model_or_obj_perms:container.view_containerrepository", + "has_file_repo_perms:file.view_filerepository", ], }, { @@ -946,8 +950,12 @@ def build_image(self, request, pk): containerfile.touch() tag = serializer.validated_data["tag"] - artifacts = serializer.validated_data["artifacts"] - Artifact.objects.filter(pk__in=artifacts.keys()).touch() + artifacts, repo_version = None, None + if serializer.validated_data.get("artifacts"): + artifacts = serializer.validated_data["artifacts"] + Artifact.objects.filter(pk__in=artifacts.keys()).touch() + elif serializer.validated_data.get("repo_version"): + repo_version = serializer.validated_data["repo_version"] result = dispatch( tasks.build_image_from_containerfile, @@ -957,6 +965,7 @@ def build_image(self, request, pk): "tag": tag, "repository_pk": str(repository.pk), "artifacts": artifacts, + "repo_version": repo_version, }, ) return OperationPostponedResponse(result, request) diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py index 5569dfa31..be71a433b 100644 --- a/pulp_container/tests/functional/api/test_build_images.py +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -9,6 +9,7 @@ from pulp_smash.pulp3.bindings import monitor_task from pulpcore.client.pulp_container import ( + ApiException, ContainerContainerDistribution, ContainerContainerRepository, ) @@ -19,7 +20,7 @@ def containerfile_name(): """A fixture for a basic container file used for building images.""" with NamedTemporaryFile() as containerfile: containerfile.write( - b"""FROM busybox:latest + b"""FROM quay.io/quay/busybox:latest # Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. COPY foo/bar/example.txt /tmp/inside-image.txt # Print the content of the file when the container starts @@ -29,7 +30,51 @@ def containerfile_name(): yield containerfile.name -def test_build_image( +@pytest.fixture +def create_file_and_container_repos_with_sample_data( + container_repository_api, + file_bindings, + file_repository_factory, + gen_object_with_cleanup, + tmp_path_factory, +): + container_repo = gen_object_with_cleanup( + container_repository_api, ContainerContainerRepository(**gen_repo()) + ) + + filename = tmp_path_factory.mktemp("fixtures") / "example.txt" + filename.write_bytes(b"test content") + file_repo = file_repository_factory(autopublish=True) + upload_task = file_bindings.ContentFilesApi.create( + relative_path="foo/bar/example.txt", file=filename, repository=file_repo.pulp_href + ).task + monitor_task(upload_task) + + return container_repo, file_repo + + +@pytest.fixture +def build_image(container_repository_api): + def _build_image(repository, containerfile, artifacts=None, repo_version=None): + # workaround for drf-spectacular not allowing artifacts=None and raising an exception + # (a bytes-like object is required, not 'NoneType') + if artifacts: + return container_repository_api.build_image( + container_container_repository_href=repository, + containerfile=containerfile, + artifacts=artifacts, + ) + return container_repository_api.build_image( + container_container_repository_href=repository, + containerfile=containerfile, + repo_version=repo_version, + ) + + return _build_image + + +def test_build_image_from_artifact( + build_image, pulpcore_bindings, container_repository_api, container_distribution_api, @@ -37,7 +82,7 @@ def test_build_image( containerfile_name, local_registry, ): - """Test if a user can build an OCI image.""" + """Test build an OCI image from an artifact.""" with NamedTemporaryFile() as text_file: text_file.write(b"some text") text_file.flush() @@ -48,7 +93,7 @@ def test_build_image( ) artifacts = '{{"{}": "foo/bar/example.txt"}}'.format(artifact.pulp_href) - build_response = container_repository_api.build_image( + build_response = build_image( repository.pulp_href, containerfile=containerfile_name, artifacts=artifacts ) monitor_task(build_response.task) @@ -61,3 +106,77 @@ def test_build_image( local_registry.pull(distribution.base_path) image = local_registry.inspect(distribution.base_path) assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] + + +def test_build_image_from_repo_version( + build_image, + containerfile_name, + container_distribution_api, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_object_with_cleanup, + local_registry, +): + """Test build an OCI image from a file repository_version.""" + container_repo, file_repo = create_file_and_container_repos_with_sample_data + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", + ) + + distribution = gen_object_with_cleanup( + container_distribution_api, + ContainerContainerDistribution(**gen_distribution(repository=container_repo.pulp_href)), + ) + + local_registry.pull(distribution.base_path) + image = local_registry.inspect(distribution.base_path) + assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] + + +def test_build_image_from_repo_version_with_anon_user( + build_image, + containerfile_name, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_user, +): + """Test if a user without permission to file repo can build an OCI image.""" + user_helpless = gen_user( + model_roles=[ + "container.containerdistribution_collaborator", + "container.containerrepository_content_manager", + ] + ) + container_repo, file_repo = create_file_and_container_repos_with_sample_data + with user_helpless, pytest.raises(ApiException): + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", + ) + + +def test_build_image_from_repo_version_with_creator_user( + build_image, + containerfile_name, + create_file_and_container_repos_with_sample_data, + delete_orphans_pre, + gen_user, +): + """Test if a user (with the expected permissions) can build an OCI image.""" + user = gen_user( + model_roles=[ + "container.containerdistribution_collaborator", + "container.containerrepository_content_manager", + "file.filerepository_viewer", + ] + ) + container_repo, file_repo = create_file_and_container_repos_with_sample_data + with user: + build_image( + container_repo.pulp_href, + containerfile_name, + repo_version=f"{file_repo.pulp_href}versions/1/", + ) diff --git a/staging_docs/admin/guides/build-image.md b/staging_docs/admin/guides/build-image.md index b71f69e9c..433f1d0ee 100644 --- a/staging_docs/admin/guides/build-image.md +++ b/staging_docs/admin/guides/build-image.md @@ -40,13 +40,23 @@ CMD ["cat", "/inside-image.txt"]' >> Containerfile ## Build an OCI image +### From artifact + ```bash TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \ artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}" | jq -r '.task') ``` -!!! warning - Non-staff users, lacking read access to the `artifacts` endpoint, may encounter restricted - functionality as they are prohibited from listing artifacts uploaded to Pulp and utilizing - them within the build process. +### From repository_version + +```bash +ARTIFACT_SHA256=$(http :$ARTIFACT_HREF | jq -r '.sha256') +pulp file repository create --name bar + +REPO_VERSION=$(pulp file content create --relative-path foo/bar/example.txt \ +--sha256 $ARTIFACT_SHA256 --repository bar | jq .pulp_href) + +TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' "containerfile@./Containerfile" \ +repo_version=$REPO_VERSION | jq -r '.task') +```