Add repository_version param as a building context

closes: pulp#479

CHANGES/479.feature

@@ -0,0 +1,4 @@
+Replaced `artifacts` by `build_context` (i.e., a file plugin repository version HREF)
+as the parameter to provide the build context for a Containerfile.
+Added the `containerfile_name` as the parameter to provide the relative-path of a File Content
+from the provided `build_context`.

CHANGES/479.removal

@@ -0,0 +1 @@
+Removed support for using raw artifacts as the build context and for the Containerfile.

docs/admin/guides/build-image.md

@@ -7,46 +7,70 @@
 
 Users can add new images to a container repository by uploading a Containerfile. The syntax for
 Containerfile is the same as for a Dockerfile. The same REST API endpoint also accepts a JSON
-string that maps artifacts in Pulp to a filename. Any artifacts passed in are available inside the
-build container at `/pulp_working_directory`.
+string that maps artifacts in Pulp to a filename. Any files passed in (via `build_context`) are
+available inside the build container at the path defined in File Content `relative-path`.
 
-## Create a Repository
+It is possible to define the Containerfile in two ways:
+* from a [local file](site:pulp_container/docs/admin/guides/build-image#build-from-a-containerfile-uploaded-during-build-request) and pass it during build request
+* from an [existing file](site:pulp_container/docs/admin/guides/build-image#upload-the-containerfile-as-a-file-content) in the `build_context`
+
+## Create a Container Repository
 
 ```bash
-REPO_HREF=$(pulp container repository create --name building | jq -r '.pulp_href')
+CONTAINER_REPO=$(pulp container repository create --name building | jq -r '.pulp_href')
 ```
 
-## Create an Artifact
+## Create a File Repository and populate it
 
 ```bash
+FILE_REPO=$(pulp file repository create --name bar --autopublish | jq -r '.pulp_href')
+
 echo 'Hello world!' > example.txt
 
-ARTIFACT_HREF=$(http --form POST http://localhost/pulp/api/v3/artifacts/ \
-    file@./example.txt \
-    | jq -r '.pulp_href')
+pulp file content upload --relative-path foo/bar/example.txt \
+--file ./example.txt --repository bar
 ```
 
 ## Create a Containerfile
 
 ```bash
 echo 'FROM centos:7
 
-# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter.
+# Copy a file using COPY statement. Use the path specified in the '--relative-path' parameter.
 COPY foo/bar/example.txt /inside-image.txt
 
 # Print the content of the file when the container starts
 CMD ["cat", "/inside-image.txt"]' >> Containerfile
 ```
 
-## Build an OCI image
+
+## Build from a Containerfile uploaded during build request
+
+### Build an OCI image with the "local" Containerfile
 
 ```bash
-TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \
-artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}"  | jq -r '.task')
+TASK_HREF=$(http --form POST :$CONTAINER_REPO'build_image/' "containerfile@./Containerfile" \
+build_context=${FILE_REPO}versions/1/ | jq -r '.task')
 ```
 
+
+## Upload the Containerfile to a File Repository and use it to build
+
+### Upload the Containerfile as a File Content
+
+```bash
+pulp file content upload --relative-path MyContainerfile --file ./Containerfile --repository bar
+```
+
+### Build an OCI image from a Containerfile present in build_context
+
+```bash
+TASK_HREF=$(http --form POST :$CONTAINER_REPO'build_image/' containerfile_name=MyContainerfile \
+build_context=${FILE_REPO}versions/2/ | jq -r '.task')
+```
+
+
 !!! warning
 
-    Non-staff users, lacking read access to the `artifacts` endpoint, may encounter restricted
-    functionality as they are prohibited from listing artifacts uploaded to Pulp and utilizing
-    them within the build process.
+    File repositories synced with on-demand policy will not automatically pull the missing artifacts.
+    Trying to build using a file that is not yet pulled will fail.

pulp_container/app/serializers.py

@@ -1,5 +1,4 @@
 from gettext import gettext as _
-import os
 import re
 
 from django.core.validators import URLValidator
@@ -758,13 +757,12 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer):
     A repository must be specified, to which the container image content will be added.
     """
 
-    containerfile_artifact = RelatedField(
-        many=False,
-        lookup_field="pk",
-        view_name="artifacts-detail",
-        queryset=Artifact.objects.all(),
+    containerfile_name = serializers.CharField(
+        required=False,
+        allow_blank=True,
         help_text=_(
-            "Artifact representing the Containerfile that should be used to run podman-build."
+            "Name of the Containerfile, from build_context, that should be used to run "
+            "podman-build."
         ),
     )
     containerfile = serializers.FileField(
@@ -774,56 +772,40 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer):
     tag = serializers.CharField(
         required=False, default="latest", help_text="A tag name for the new image being built."
     )
-    artifacts = serializers.JSONField(
+    build_context = RepositoryVersionRelatedField(
         required=False,
-        help_text="A JSON string where each key is an artifact href and the value is it's "
-        "relative path (name) inside the /pulp_working_directory of the build container "
-        "executing the Containerfile.",
+        help_text=_("RepositoryVersion to be used as the build context for container images."),
+        allow_null=True,
+        queryset=RepositoryVersion.objects.filter(repository__pulp_type="file.file"),
     )
 
     def __init__(self, *args, **kwargs):
         """Initializer for OCIBuildImageSerializer."""
         super().__init__(*args, **kwargs)
-        self.fields["containerfile_artifact"].required = False
 
     def validate(self, data):
         """Validates that all the fields make sense."""
         data = super().validate(data)
 
+        if bool(data.get("containerfile", None)) == bool(data.get("containerfile_name", None)):
+            raise serializers.ValidationError(
+                _("'containerfile' or 'containerfile_name' must be specified.")
+            )
+
         if "containerfile" in data:
-            if "containerfile_artifact" in data:
-                raise serializers.ValidationError(
-                    _("Only one of 'containerfile' and 'containerfile_artifact' may be specified.")
-                )
             data["containerfile_artifact"] = Artifact.init_and_validate(data.pop("containerfile"))
-        elif "containerfile_artifact" in data:
-            data["containerfile_artifact"].touch()
-        else:
+
+        if "containerfile_name" in data and "build_context" not in data:
             raise serializers.ValidationError(
-                _("'containerfile' or 'containerfile_artifact' must " "be specified.")
+                _("A 'build_context' must be specified when 'containerfile_name' is present.")
             )
-        artifacts = {}
-        if "artifacts" in data:
-            for url, relative_path in data["artifacts"].items():
-                if os.path.isabs(relative_path):
-                    raise serializers.ValidationError(
-                        _("Relative path cannot start with '/'. " "{0}").format(relative_path)
-                    )
-                artifactfield = RelatedField(
-                    view_name="artifacts-detail",
-                    queryset=Artifact.objects.all(),
-                    source="*",
-                    initial=url,
-                )
-                try:
-                    artifact = artifactfield.run_validation(data=url)
-                    artifact.touch()
-                    artifacts[str(artifact.pk)] = relative_path
-                except serializers.ValidationError as e:
-                    # Append the URL of missing Artifact to the error message
-                    e.detail[0] = "%s %s" % (e.detail[0], url)
-                    raise e
-        data["artifacts"] = artifacts
+
+        # the "has_repo_or_repo_ver_param_model_or_obj_perms" permission condition function expects
+        # a "repo" or "repository_version" arguments, so we need to pass "build_context" as
+        # "repository_version" to be able to validate the permissions
+        if data.get("build_context", None):
+            data["repository_version"] = data["build_context"]
+
         return data
 
     class Meta:
@@ -832,7 +814,7 @@ class Meta:
             "containerfile",
             "repository",
             "tag",
-            "artifacts",
+            "build_context",
         )
 
 

pulp_container/app/tasks/builder.py

@@ -14,7 +14,7 @@
 )
 from pulp_container.constants import MEDIA_TYPE
 from pulp_container.app.utils import calculate_digest
-from pulpcore.plugin.models import Artifact, ContentArtifact, Content
+from pulpcore.plugin.models import Artifact, ContentArtifact, Content, RepositoryVersion
 
 
 def get_or_create_blob(layer_json, manifest, path):
@@ -96,7 +96,11 @@ def add_image_from_directory_to_repository(path, repository, tag):
 
 
 def build_image_from_containerfile(
-    containerfile_pk=None, artifacts=None, repository_pk=None, tag=None
+    containerfile_pk=None,
+    build_context_pk=None,
+    repository_pk=None,
+    tag=None,
+    containerfile_name=None,
 ):
     """
     Builds an OCI container image from a Containerfile.
@@ -106,34 +110,52 @@ def build_image_from_containerfile(
 
     Args:
         containerfile_pk (str): The pk of an Artifact that contains the Containerfile
-        artifacts (dict): A dictionary where each key is an artifact PK and the value is it's
-                          relative path (name) inside the /pulp_working_directory of the build
-                          container executing the Containerfile.
         repository_pk (str): The pk of a Repository to add the OCI container image
         tag (str): Tag name for the new image in the repository
+        build_context_pk: The pk of a RepositoryVersion with the artifacts used in the build context
+                      of the Containerfile.
+        containerfile_name: Name of the Containerfile, stored as a File Content, from build_context
 
     Returns:
         A class:`pulpcore.plugin.models.RepositoryVersion` that contains the new OCI container
         image and tag.
 
     """
-    containerfile = Artifact.objects.get(pk=containerfile_pk)
+    if containerfile_pk:
+        containerfile = Artifact.objects.get(pk=containerfile_pk)
     repository = ContainerRepository.objects.get(pk=repository_pk)
     name = str(uuid4())
     with tempfile.TemporaryDirectory(dir=".") as working_directory:
         working_directory = os.path.abspath(working_directory)
         context_path = os.path.join(working_directory, "context")
         os.makedirs(context_path, exist_ok=True)
-        for key, val in artifacts.items():
-            artifact = Artifact.objects.get(pk=key)
-            dest_path = os.path.join(context_path, val)
-            dirs = os.path.split(dest_path)[0]
-            if dirs:
-                os.makedirs(dirs, exist_ok=True)
-            with open(dest_path, "wb") as dest:
-                shutil.copyfileobj(artifact.file, dest)
 
-            containerfile_path = os.path.join(working_directory, "Containerfile")
+        if build_context_pk:
+            build_context = RepositoryVersion.objects.get(pk=build_context_pk)
+            content_artifacts = ContentArtifact.objects.filter(
+                content__pulp_type="file.file", content__in=build_context.content
+            ).order_by("-content__pulp_created")
+            for content_artifact in content_artifacts.select_related("artifact").iterator():
+                if not content_artifact.artifact:
+                    raise RuntimeError(
+                        "It is not possible to use File content synced with on-demand "
+                        "policy without pulling the data first."
+                    )
+                if containerfile_name and content_artifact.relative_path == containerfile_name:
+                    containerfile = Artifact.objects.get(pk=content_artifact.artifact.pk)
+                    continue
+                _copy_file_from_artifact(
+                    context_path, content_artifact.relative_path, content_artifact.artifact.file
+                )
+
+        try:
+            containerfile
+        except NameError:
+            raise RuntimeError(
+                '"' + containerfile_name + '" containerfile not found in build_context!'
+            )
+
+        containerfile_path = os.path.join(working_directory, "Containerfile")
 
         with open(containerfile_path, "wb") as dest:
             shutil.copyfileobj(containerfile.file, dest)
@@ -166,3 +188,12 @@ def build_image_from_containerfile(
         repository_version = add_image_from_directory_to_repository(image_dir, repository, tag)
 
     return repository_version
+
+
+def _copy_file_from_artifact(context_path, relative_path, artifact):
+    dest_path = os.path.join(context_path, relative_path)
+    dirs = os.path.dirname(dest_path)
+    if dirs:
+        os.makedirs(dirs, exist_ok=True)
+    with open(dest_path, "wb") as dest:
+        shutil.copyfileobj(artifact.file, dest)

pulp_container/app/viewsets.py

@@ -696,6 +696,7 @@ class ContainerRepositoryViewSet(
                 "condition": [
                     "has_model_or_obj_perms:container.build_image_containerrepository",
                     "has_model_or_obj_perms:container.view_containerrepository",
+                    "has_repo_or_repo_ver_param_model_or_obj_perms:file.view_filerepository",
                 ],
             },
             {
@@ -938,25 +939,31 @@ def build_image(self, request, pk):
 
         serializer.is_valid(raise_exception=True)
 
-        containerfile = serializer.validated_data["containerfile_artifact"]
-        try:
-            containerfile.save()
-        except IntegrityError:
-            containerfile = Artifact.objects.get(sha256=containerfile.sha256)
-            containerfile.touch()
+        containerfile_pk = None
+        if containerfile := serializer.validated_data.get("containerfile_artifact", None):
+            try:
+                containerfile.save()
+            except IntegrityError:
+                containerfile = Artifact.objects.get(sha256=containerfile.sha256)
+                containerfile.touch()
+            containerfile_pk = str(containerfile.pk)
+
         tag = serializer.validated_data["tag"]
+        containerfile_name = serializer.validated_data.get("containerfile_name", None)
 
-        artifacts = serializer.validated_data["artifacts"]
-        Artifact.objects.filter(pk__in=artifacts.keys()).touch()
+        build_context_pk = None
+        if build_context := serializer.validated_data.get("build_context", None):
+            build_context_pk = build_context.pk
 
         result = dispatch(
             tasks.build_image_from_containerfile,
             exclusive_resources=[repository],
             kwargs={
-                "containerfile_pk": str(containerfile.pk),
+                "containerfile_name": containerfile_name,
+                "containerfile_pk": containerfile_pk,
                 "tag": tag,
                 "repository_pk": str(repository.pk),
-                "artifacts": artifacts,
+                "build_context_pk": build_context_pk,
             },
         )
         return OperationPostponedResponse(result, request)