From 3aa85a4463ca52aec5a50ba6fa83309a818b3698 Mon Sep 17 00:00:00 2001 From: git-hyagi <45576767+git-hyagi@users.noreply.github.com> Date: Tue, 25 Jun 2024 16:54:03 -0300 Subject: [PATCH] Add repository_version param as a building context closes: #479 --- CHANGES/479.feature | 4 + CHANGES/479.removal | 1 + docs/admin/guides/build-image.md | 54 ++++-- pulp_container/app/serializers.py | 68 +++---- pulp_container/app/tasks/builder.py | 61 ++++-- pulp_container/app/viewsets.py | 27 ++- .../tests/functional/api/test_build_images.py | 182 +++++++++++++++--- 7 files changed, 288 insertions(+), 109 deletions(-) create mode 100644 CHANGES/479.feature create mode 100644 CHANGES/479.removal diff --git a/CHANGES/479.feature b/CHANGES/479.feature new file mode 100644 index 000000000..2bf1dc347 --- /dev/null +++ b/CHANGES/479.feature @@ -0,0 +1,4 @@ +Replaced `artifacts` by `build_context` (i.e., a file plugin repository version HREF) +as the parameter to provide the build context for a Containerfile. +Added the `containerfile_name` as the parameter to provide the relative-path of a File Content +from the provided `build_context`. diff --git a/CHANGES/479.removal b/CHANGES/479.removal new file mode 100644 index 000000000..7b2f81b55 --- /dev/null +++ b/CHANGES/479.removal @@ -0,0 +1 @@ +Removed support for using raw artifacts as the build context and for the Containerfile. diff --git a/docs/admin/guides/build-image.md b/docs/admin/guides/build-image.md index b71f69e9c..809d7b82c 100644 --- a/docs/admin/guides/build-image.md +++ b/docs/admin/guides/build-image.md @@ -7,23 +7,28 @@ Users can add new images to a container repository by uploading a Containerfile. The syntax for Containerfile is the same as for a Dockerfile. The same REST API endpoint also accepts a JSON -string that maps artifacts in Pulp to a filename. Any artifacts passed in are available inside the -build container at `/pulp_working_directory`. +string that maps artifacts in Pulp to a filename. Any files passed in (via `build_context`) are +available inside the build container at the path defined in File Content `relative-path`. -## Create a Repository +It is possible to define the Containerfile in two ways: +* from a [local file](site:pulp_container/docs/admin/guides/build-image#build-from-a-containerfile-uploaded-during-build-request) and pass it during build request +* from an [existing file](site:pulp_container/docs/admin/guides/build-image#upload-the-containerfile-as-a-file-content) in the `build_context` + +## Create a Container Repository ```bash -REPO_HREF=$(pulp container repository create --name building | jq -r '.pulp_href') +CONTAINER_REPO=$(pulp container repository create --name building | jq -r '.pulp_href') ``` -## Create an Artifact +## Create a File Repository and populate it ```bash +FILE_REPO=$(pulp file repository create --name bar --autopublish | jq -r '.pulp_href') + echo 'Hello world!' > example.txt -ARTIFACT_HREF=$(http --form POST http://localhost/pulp/api/v3/artifacts/ \ - file@./example.txt \ - | jq -r '.pulp_href') +pulp file content upload --relative-path foo/bar/example.txt \ +--file ./example.txt --repository bar ``` ## Create a Containerfile @@ -31,22 +36,41 @@ ARTIFACT_HREF=$(http --form POST http://localhost/pulp/api/v3/artifacts/ \ ```bash echo 'FROM centos:7 -# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. +# Copy a file using COPY statement. Use the path specified in the '--relative-path' parameter. COPY foo/bar/example.txt /inside-image.txt # Print the content of the file when the container starts CMD ["cat", "/inside-image.txt"]' >> Containerfile ``` -## Build an OCI image + +## Build from a Containerfile uploaded during build request + +### Build an OCI image with the "local" Containerfile ```bash -TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \ -artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}" | jq -r '.task') +TASK_HREF=$(http --form POST :$CONTAINER_REPO'build_image/' "containerfile@./Containerfile" \ +build_context=${FILE_REPO}versions/1/ | jq -r '.task') ``` + +## Upload the Containerfile to a File Repository and use it to build + +### Upload the Containerfile as a File Content + +```bash +pulp file content upload --relative-path MyContainerfile --file ./Containerfile --repository bar +``` + +### Build an OCI image from a Containerfile present in build_context + +```bash +TASK_HREF=$(http --form POST :$CONTAINER_REPO'build_image/' containerfile_name=MyContainerfile \ +build_context=${FILE_REPO}versions/2/ | jq -r '.task') +``` + + !!! warning - Non-staff users, lacking read access to the `artifacts` endpoint, may encounter restricted - functionality as they are prohibited from listing artifacts uploaded to Pulp and utilizing - them within the build process. + File repositories synced with on-demand policy will not automatically pull the missing artifacts. + Trying to build using a file that is not yet pulled will fail. diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index d2922bf18..ff869e06e 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -1,5 +1,4 @@ from gettext import gettext as _ -import os import re from django.core.validators import URLValidator @@ -758,13 +757,12 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer): A repository must be specified, to which the container image content will be added. """ - containerfile_artifact = RelatedField( - many=False, - lookup_field="pk", - view_name="artifacts-detail", - queryset=Artifact.objects.all(), + containerfile_name = serializers.CharField( + required=False, + allow_blank=True, help_text=_( - "Artifact representing the Containerfile that should be used to run podman-build." + "Name of the Containerfile, from build_context, that should be used to run " + "podman-build." ), ) containerfile = serializers.FileField( @@ -774,56 +772,40 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer): tag = serializers.CharField( required=False, default="latest", help_text="A tag name for the new image being built." ) - artifacts = serializers.JSONField( + build_context = RepositoryVersionRelatedField( required=False, - help_text="A JSON string where each key is an artifact href and the value is it's " - "relative path (name) inside the /pulp_working_directory of the build container " - "executing the Containerfile.", + help_text=_("RepositoryVersion to be used as the build context for container images."), + allow_null=True, + queryset=RepositoryVersion.objects.filter(repository__pulp_type="file.file"), ) def __init__(self, *args, **kwargs): """Initializer for OCIBuildImageSerializer.""" super().__init__(*args, **kwargs) - self.fields["containerfile_artifact"].required = False def validate(self, data): """Validates that all the fields make sense.""" data = super().validate(data) + if bool(data.get("containerfile", None)) == bool(data.get("containerfile_name", None)): + raise serializers.ValidationError( + _("'containerfile' or 'containerfile_name' must be specified.") + ) + if "containerfile" in data: - if "containerfile_artifact" in data: - raise serializers.ValidationError( - _("Only one of 'containerfile' and 'containerfile_artifact' may be specified.") - ) data["containerfile_artifact"] = Artifact.init_and_validate(data.pop("containerfile")) - elif "containerfile_artifact" in data: - data["containerfile_artifact"].touch() - else: + + if "containerfile_name" in data and "build_context" not in data: raise serializers.ValidationError( - _("'containerfile' or 'containerfile_artifact' must " "be specified.") + _("A 'build_context' must be specified when 'containerfile_name' is present.") ) - artifacts = {} - if "artifacts" in data: - for url, relative_path in data["artifacts"].items(): - if os.path.isabs(relative_path): - raise serializers.ValidationError( - _("Relative path cannot start with '/'. " "{0}").format(relative_path) - ) - artifactfield = RelatedField( - view_name="artifacts-detail", - queryset=Artifact.objects.all(), - source="*", - initial=url, - ) - try: - artifact = artifactfield.run_validation(data=url) - artifact.touch() - artifacts[str(artifact.pk)] = relative_path - except serializers.ValidationError as e: - # Append the URL of missing Artifact to the error message - e.detail[0] = "%s %s" % (e.detail[0], url) - raise e - data["artifacts"] = artifacts + + # the "has_repo_or_repo_ver_param_model_or_obj_perms" permission condition function expects + # a "repo" or "repository_version" arguments, so we need to pass "build_context" as + # "repository_version" to be able to validate the permissions + if data.get("build_context", None): + data["repository_version"] = data["build_context"] + return data class Meta: @@ -832,7 +814,7 @@ class Meta: "containerfile", "repository", "tag", - "artifacts", + "build_context", ) diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 58daa3796..a1c0dace1 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -14,7 +14,7 @@ ) from pulp_container.constants import MEDIA_TYPE from pulp_container.app.utils import calculate_digest -from pulpcore.plugin.models import Artifact, ContentArtifact, Content +from pulpcore.plugin.models import Artifact, ContentArtifact, Content, RepositoryVersion def get_or_create_blob(layer_json, manifest, path): @@ -96,7 +96,11 @@ def add_image_from_directory_to_repository(path, repository, tag): def build_image_from_containerfile( - containerfile_pk=None, artifacts=None, repository_pk=None, tag=None + containerfile_pk=None, + build_context_pk=None, + repository_pk=None, + tag=None, + containerfile_name=None, ): """ Builds an OCI container image from a Containerfile. @@ -106,34 +110,52 @@ def build_image_from_containerfile( Args: containerfile_pk (str): The pk of an Artifact that contains the Containerfile - artifacts (dict): A dictionary where each key is an artifact PK and the value is it's - relative path (name) inside the /pulp_working_directory of the build - container executing the Containerfile. repository_pk (str): The pk of a Repository to add the OCI container image tag (str): Tag name for the new image in the repository + build_context_pk: The pk of a RepositoryVersion with the artifacts used in the build context + of the Containerfile. + containerfile_name: Name of the Containerfile, stored as a File Content, from build_context Returns: A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container image and tag. """ - containerfile = Artifact.objects.get(pk=containerfile_pk) + if containerfile_pk: + containerfile = Artifact.objects.get(pk=containerfile_pk) repository = ContainerRepository.objects.get(pk=repository_pk) name = str(uuid4()) with tempfile.TemporaryDirectory(dir=".") as working_directory: working_directory = os.path.abspath(working_directory) context_path = os.path.join(working_directory, "context") os.makedirs(context_path, exist_ok=True) - for key, val in artifacts.items(): - artifact = Artifact.objects.get(pk=key) - dest_path = os.path.join(context_path, val) - dirs = os.path.split(dest_path)[0] - if dirs: - os.makedirs(dirs, exist_ok=True) - with open(dest_path, "wb") as dest: - shutil.copyfileobj(artifact.file, dest) - containerfile_path = os.path.join(working_directory, "Containerfile") + if build_context_pk: + build_context = RepositoryVersion.objects.get(pk=build_context_pk) + content_artifacts = ContentArtifact.objects.filter( + content__pulp_type="file.file", content__in=build_context.content + ).order_by("-content__pulp_created") + for content_artifact in content_artifacts.select_related("artifact").iterator(): + if not content_artifact.artifact: + raise RuntimeError( + "It is not possible to use File content synced with on-demand " + "policy without pulling the data first." + ) + if containerfile_name and content_artifact.relative_path == containerfile_name: + containerfile = Artifact.objects.get(pk=content_artifact.artifact.pk) + continue + _copy_file_from_artifact( + context_path, content_artifact.relative_path, content_artifact.artifact.file + ) + + try: + containerfile + except NameError: + raise RuntimeError( + '"' + containerfile_name + '" containerfile not found in build_context!' + ) + + containerfile_path = os.path.join(working_directory, "Containerfile") with open(containerfile_path, "wb") as dest: shutil.copyfileobj(containerfile.file, dest) @@ -166,3 +188,12 @@ def build_image_from_containerfile( repository_version = add_image_from_directory_to_repository(image_dir, repository, tag) return repository_version + + +def _copy_file_from_artifact(context_path, relative_path, artifact): + dest_path = os.path.join(context_path, relative_path) + dirs = os.path.dirname(dest_path) + if dirs: + os.makedirs(dirs, exist_ok=True) + with open(dest_path, "wb") as dest: + shutil.copyfileobj(artifact.file, dest) diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index 8ae0bdcf4..cd7ecf93f 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -696,6 +696,7 @@ class ContainerRepositoryViewSet( "condition": [ "has_model_or_obj_perms:container.build_image_containerrepository", "has_model_or_obj_perms:container.view_containerrepository", + "has_repo_or_repo_ver_param_model_or_obj_perms:file.view_filerepository", ], }, { @@ -938,25 +939,31 @@ def build_image(self, request, pk): serializer.is_valid(raise_exception=True) - containerfile = serializer.validated_data["containerfile_artifact"] - try: - containerfile.save() - except IntegrityError: - containerfile = Artifact.objects.get(sha256=containerfile.sha256) - containerfile.touch() + containerfile_pk = None + if containerfile := serializer.validated_data.get("containerfile_artifact", None): + try: + containerfile.save() + except IntegrityError: + containerfile = Artifact.objects.get(sha256=containerfile.sha256) + containerfile.touch() + containerfile_pk = str(containerfile.pk) + tag = serializer.validated_data["tag"] + containerfile_name = serializer.validated_data.get("containerfile_name", None) - artifacts = serializer.validated_data["artifacts"] - Artifact.objects.filter(pk__in=artifacts.keys()).touch() + build_context_pk = None + if build_context := serializer.validated_data.get("build_context", None): + build_context_pk = build_context.pk result = dispatch( tasks.build_image_from_containerfile, exclusive_resources=[repository], kwargs={ - "containerfile_pk": str(containerfile.pk), + "containerfile_name": containerfile_name, + "containerfile_pk": containerfile_pk, "tag": tag, "repository_pk": str(repository.pk), - "artifacts": artifacts, + "build_context_pk": build_context_pk, }, ) return OperationPostponedResponse(result, request) diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py index 5569dfa31..ba5cafda1 100644 --- a/pulp_container/tests/functional/api/test_build_images.py +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -2,16 +2,11 @@ from tempfile import NamedTemporaryFile -from pulp_smash.pulp3.utils import ( - gen_distribution, - gen_repo, -) +from pulpcore.tests.functional.utils import PulpTaskError +from pulp_smash.pulp3.utils import gen_distribution from pulp_smash.pulp3.bindings import monitor_task -from pulpcore.client.pulp_container import ( - ContainerContainerDistribution, - ContainerContainerRepository, -) +from pulpcore.client.pulp_container import ApiException, ContainerContainerDistribution @pytest.fixture @@ -19,7 +14,7 @@ def containerfile_name(): """A fixture for a basic container file used for building images.""" with NamedTemporaryFile() as containerfile: containerfile.write( - b"""FROM busybox:latest + b"""FROM quay.io/quay/busybox:latest # Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter. COPY foo/bar/example.txt /tmp/inside-image.txt # Print the content of the file when the container starts @@ -29,33 +24,168 @@ def containerfile_name(): yield containerfile.name -def test_build_image( - pulpcore_bindings, - container_repository_api, +@pytest.fixture +def populated_file_repo( + containerfile_name, + file_bindings, + file_repo, + tmp_path_factory, +): + filename = tmp_path_factory.mktemp("fixtures") / "example.txt" + filename.write_bytes(b"test content") + upload_task = file_bindings.ContentFilesApi.create( + relative_path="foo/bar/example.txt", file=filename, repository=file_repo.pulp_href + ).task + monitor_task(upload_task) + + upload_task = file_bindings.ContentFilesApi.create( + relative_path="Containerfile", file=containerfile_name, repository=file_repo.pulp_href + ).task + monitor_task(upload_task) + + return file_repo + + +@pytest.fixture +def build_image(container_repository_api): + def _build_image(repository, containerfile=None, containerfile_name=None, build_context=None): + if not containerfile_name: + containerfile_name = "" + + if not build_context: + build_context = b"" + + build_response = container_repository_api.build_image( + container_container_repository_href=repository, + containerfile=containerfile, + containerfile_name=containerfile_name, + build_context=build_context, + ) + monitor_task(build_response.task) + + return _build_image + + +def test_build_image_with_uploaded_containerfile( + build_image, + containerfile_name, container_distribution_api, + container_repo, + populated_file_repo, + delete_orphans_pre, gen_object_with_cleanup, - containerfile_name, local_registry, ): - """Test if a user can build an OCI image.""" - with NamedTemporaryFile() as text_file: - text_file.write(b"some text") - text_file.flush() - artifact = gen_object_with_cleanup(pulpcore_bindings.ArtifactsApi, text_file.name) - - repository = gen_object_with_cleanup( - container_repository_api, ContainerContainerRepository(**gen_repo()) + """Test build an OCI image from a file repository_version.""" + build_image( + repository=container_repo.pulp_href, + containerfile=containerfile_name, + build_context=f"{populated_file_repo.pulp_href}versions/1/", + ) + + distribution = gen_object_with_cleanup( + container_distribution_api, + ContainerContainerDistribution(**gen_distribution(repository=container_repo.pulp_href)), + ) + + local_registry.pull(distribution.base_path) + image = local_registry.inspect(distribution.base_path) + assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"] + + +def test_build_image_from_repo_version_with_anon_user( + build_image, + containerfile_name, + container_repo, + delete_orphans_pre, + populated_file_repo, + gen_user, +): + """Test if a user without permission to file repo can build an OCI image.""" + user_helpless = gen_user( + model_roles=[ + "container.containerdistribution_collaborator", + "container.containerrepository_content_manager", + ] ) + with user_helpless, pytest.raises(ApiException): + build_image( + container_repo.pulp_href, + containerfile_name, + build_context=f"{populated_file_repo.pulp_href}versions/1/", + ) + - artifacts = '{{"{}": "foo/bar/example.txt"}}'.format(artifact.pulp_href) - build_response = container_repository_api.build_image( - repository.pulp_href, containerfile=containerfile_name, artifacts=artifacts +def test_build_image_from_repo_version_with_creator_user( + build_image, + containerfile_name, + container_repo, + delete_orphans_pre, + populated_file_repo, + gen_user, +): + """Test if a user (with the expected permissions) can build an OCI image.""" + user = gen_user( + object_roles=[ + ("container.containerrepository_content_manager", container_repo.pulp_href), + ("file.filerepository_viewer", populated_file_repo.pulp_href), + ], + ) + with user: + build_image( + container_repo.pulp_href, + containerfile_name, + build_context=f"{populated_file_repo.pulp_href}versions/1/", + ) + + +def test_build_image_without_containerfile( + build_image, + container_repo, + populated_file_repo, +): + """Test build an OCI image without a containerfile""" + with pytest.raises(ApiException): + build_image( + repository=container_repo.pulp_href, + build_context=f"{populated_file_repo.pulp_href}versions/2/", + ) + + +def test_build_image_without_expected_files( + build_image, + containerfile_name, + container_repo, +): + """ + Test build an OCI image without the expected files (build_context) defined in the Containerfile + """ + with pytest.raises(PulpTaskError): + build_image( + repository=container_repo.pulp_href, + containerfile=containerfile_name, + ) + + +def test_build_image_from_containerfile_name( + build_image, + container_distribution_api, + container_repo, + delete_orphans_pre, + gen_object_with_cleanup, + local_registry, + populated_file_repo, +): + """Test build an OCI image with a containerfile from build_context.""" + build_image( + repository=container_repo.pulp_href, + containerfile_name="Containerfile", + build_context=f"{populated_file_repo.pulp_href}versions/2/", ) - monitor_task(build_response.task) distribution = gen_object_with_cleanup( container_distribution_api, - ContainerContainerDistribution(**gen_distribution(repository=repository.pulp_href)), + ContainerContainerDistribution(**gen_distribution(repository=container_repo.pulp_href)), ) local_registry.pull(distribution.base_path)