Add repository_version param as a building context

closes: pulp#479

CHANGES/479.feature

@@ -0,0 +1,5 @@
+Updated the image building feature:
+    * Replaced `artifacts` by `build_context` (i.e., a file plugin repository version HREF)
+    as the parameter to provide the build context for a Containerfile.
+    * Added the `containerfile_name` as the parameter to provide the relative-path of a File Content
+    from the provided `build_context`.

CHANGES/479.removal

@@ -0,0 +1,2 @@
+Support for utilizing raw artifacts, as the build context and for the Containerfile,
+during image builds has been removed.

docs/admin/guides/build-image.md

@@ -6,47 +6,78 @@
     guaranteed.
 
 Users can add new images to a container repository by uploading a Containerfile. The syntax for
-Containerfile is the same as for a Dockerfile. The same REST API endpoint also accepts a JSON
-string that maps artifacts in Pulp to a filename. Any artifacts passed in are available inside the
-build container at `/pulp_working_directory`.
+Containerfile is the same as for a Dockerfile.
 
-## Create a Repository
+To pass arbitrary files or artifacts to the image building context, the `build_context` property (a reference to a file repository) can be provided in the request payload.
+These files can be referenced in Containerfile by passing their `relative-path`:
+```
+ADD/COPY <file relative-path> <location in container>
+```
+
+It is possible to define the Containerfile in two ways:
+* from a [local file](site:pulp_container/docs/admin/guides/build-image#build-from-a-containerfile-uploaded-during-build-request) and pass it during build request
+* from an [existing file](site:pulp_container/docs/admin/guides/build-image#upload-the-containerfile-as-a-file-content) in the `build_context`
+
+## Create a Container Repository
 
 ```bash
-REPO_HREF=$(pulp container repository create --name building | jq -r '.pulp_href')
+CONTAINER_REPO=$(pulp container repository create --name building | jq -r '.pulp_href')
 ```
 
-## Create an Artifact
+## Create a File Repository and populate it
 
 ```bash
+FILE_REPO=$(pulp file repository create --name bar --autopublish | jq -r '.pulp_href')
+
 echo 'Hello world!' > example.txt
 
-ARTIFACT_HREF=$(http --form POST http://localhost/pulp/api/v3/artifacts/ \
-    file@./example.txt \
-    | jq -r '.pulp_href')
+pulp file content upload --relative-path foo/bar/example.txt \
+--file ./example.txt --repository bar
 ```
 
 ## Create a Containerfile
 
 ```bash
 echo 'FROM centos:7
 
-# Copy a file using COPY statement. Use the relative path specified in the 'artifacts' parameter.
+# Copy a file using COPY statement. Use the path specified in the '--relative-path' parameter.
 COPY foo/bar/example.txt /inside-image.txt
 
 # Print the content of the file when the container starts
 CMD ["cat", "/inside-image.txt"]' >> Containerfile
 ```
 
-## Build an OCI image
+
+## Build from a Containerfile uploaded during build request
 
 ```bash
-TASK_HREF=$(http --form POST :$REPO_HREF'build_image/' containerfile@./Containerfile \
-artifacts="{\"$ARTIFACT_HREF\": \"foo/bar/example.txt\"}"  | jq -r '.task')
+TASK_HREF=$(http --form POST ${BASE_ADDR}${CONTAINER_REPO}'build_image/' "containerfile@./Containerfile" \
+build_context=${FILE_REPO}versions/1/ | jq -r '.task')
 ```
 
+!!! note
+
+    If `TMPFILE_PROTECTION_TIME` is set to 0 (default value), the automatic cleanup is disabled,
+    meaning all the uploaded Containerfile used for the builds will be kept in `FILE_UPLOAD_TEMP_DIR`.
+
+
+## Upload the Containerfile to a File Repository and use it to build
+
+### Upload the Containerfile as a File Content
+
+```bash
+pulp file content upload --relative-path MyContainerfile --file ./Containerfile --repository bar
+```
+
+### Build an OCI image from a Containerfile present in build_context
+
+```bash
+TASK_HREF=$(http --form POST ${BASE_ADDR}${CONTAINER_REPO}'build_image/' containerfile_name=MyContainerfile \
+build_context=${FILE_REPO}versions/2/ | jq -r '.task')
+```
+
+
 !!! warning
 
-    Non-staff users, lacking read access to the `artifacts` endpoint, may encounter restricted
-    functionality as they are prohibited from listing artifacts uploaded to Pulp and utilizing
-    them within the build process.
+    File repositories synced with the on-demand policy will not automatically download the missing artifacts.
+    Trying to build an image using a file that has not yet been downloaded will fail.

pulp_container/app/serializers.py

@@ -1,12 +1,11 @@
 from gettext import gettext as _
-import os
 import re
 
 from django.core.validators import URLValidator
 from rest_framework import serializers
 
 from pulpcore.plugin.models import (
-    Artifact,
+    ContentArtifact,
     ContentRedirectContentGuard,
     Remote,
     Repository,
@@ -758,13 +757,12 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer):
     A repository must be specified, to which the container image content will be added.
     """
 
-    containerfile_artifact = RelatedField(
-        many=False,
-        lookup_field="pk",
-        view_name="artifacts-detail",
-        queryset=Artifact.objects.all(),
+    containerfile_name = serializers.CharField(
+        required=False,
+        allow_blank=True,
         help_text=_(
-            "Artifact representing the Containerfile that should be used to run podman-build."
+            "Name of the Containerfile, from build_context, that should be used to run "
+            "podman-build."
         ),
     )
     containerfile = serializers.FileField(
@@ -774,65 +772,78 @@ class OCIBuildImageSerializer(ValidateFieldsMixin, serializers.Serializer):
     tag = serializers.CharField(
         required=False, default="latest", help_text="A tag name for the new image being built."
     )
-    artifacts = serializers.JSONField(
+    build_context = RepositoryVersionRelatedField(
         required=False,
-        help_text="A JSON string where each key is an artifact href and the value is it's "
-        "relative path (name) inside the /pulp_working_directory of the build container "
-        "executing the Containerfile.",
+        help_text=_("RepositoryVersion to be used as the build context for container images."),
+        allow_null=True,
+        queryset=RepositoryVersion.objects.filter(repository__pulp_type="file.file"),
     )
 
-    def __init__(self, *args, **kwargs):
-        """Initializer for OCIBuildImageSerializer."""
-        super().__init__(*args, **kwargs)
-        self.fields["containerfile_artifact"].required = False
-
     def validate(self, data):
         """Validates that all the fields make sense."""
         data = super().validate(data)
 
-        if "containerfile" in data:
-            if "containerfile_artifact" in data:
-                raise serializers.ValidationError(
-                    _("Only one of 'containerfile' and 'containerfile_artifact' may be specified.")
-                )
-            data["containerfile_artifact"] = Artifact.init_and_validate(data.pop("containerfile"))
-        elif "containerfile_artifact" in data:
-            data["containerfile_artifact"].touch()
-        else:
+        if bool(data.get("containerfile", None)) == bool(data.get("containerfile_name", None)):
+            raise serializers.ValidationError(
+                _("Exactly one of 'containerfile' or 'containerfile_name' must be specified.")
+            )
+
+        if "containerfile_name" in data and "build_context" not in data:
             raise serializers.ValidationError(
-                _("'containerfile' or 'containerfile_artifact' must " "be specified.")
+                _("A 'build_context' must be specified when 'containerfile_name' is present.")
             )
-        artifacts = {}
-        if "artifacts" in data:
-            for url, relative_path in data["artifacts"].items():
-                if os.path.isabs(relative_path):
+
+        # TODO: this can be removed after https://github.com/pulp/pulpcore/issues/5786
+        if data.get("build_context", None):
+            data["repository_version"] = data["build_context"]
+
+        return data
+
+    def deferred_validation(self, data):
+        """
+        Defer the validation of the `build_context` parameter to avoid rerunning unnecessary
+        database queries when checking permissions (DRF Access Policy).
+        """
+        if build_context := data.get("build_context", None):
+            data["content_artifact_pks"] = []
+            repository_version = RepositoryVersion.objects.get(pk=build_context.pk)
+            content_artifacts = ContentArtifact.objects.filter(
+                content__pulp_type="file.file", content__in=repository_version.content
+            ).order_by("-content__pulp_created")
+            for content_artifact in content_artifacts.select_related("artifact").iterator():
+                if not content_artifact.artifact:
                     raise serializers.ValidationError(
-                        _("Relative path cannot start with '/'. " "{0}").format(relative_path)
+                        _(
+                            "It is not possible to use File content synced with on-demand "
+                            "policy without pulling the data first."
+                        )
+                    )
+                containerfile_name = data.get("containerfile_name", None)
+                if containerfile_name and content_artifact.relative_path == containerfile_name:
+                    data["containerfile_artifact_pk"] = str(content_artifact.artifact.pk)
+
+                data["content_artifact_pks"].append(content_artifact.pk)
+
+            if data.get("containerfile_name") and not data.get("containerfile_artifact_pk", None):
+                raise serializers.ValidationError(
+                    _(
+                        'Could not find the Containerfile "'
+                        + data["containerfile_name"]
+                        + '" in the build_context provided'
                     )
-                artifactfield = RelatedField(
-                    view_name="artifacts-detail",
-                    queryset=Artifact.objects.all(),
-                    source="*",
-                    initial=url,
                 )
-                try:
-                    artifact = artifactfield.run_validation(data=url)
-                    artifact.touch()
-                    artifacts[str(artifact.pk)] = relative_path
-                except serializers.ValidationError as e:
-                    # Append the URL of missing Artifact to the error message
-                    e.detail[0] = "%s %s" % (e.detail[0], url)
-                    raise e
-        data["artifacts"] = artifacts
+
+            data["build_context_repo"] = repository_version.repository
+
         return data
 
     class Meta:
         fields = (
-            "containerfile_artifact",
+            "containerfile_name",
             "containerfile",
             "repository",
             "tag",
-            "artifacts",
+            "build_context",
         )
 
 

pulp_container/app/tasks/builder.py

@@ -14,7 +14,12 @@
 )
 from pulp_container.constants import MEDIA_TYPE
 from pulp_container.app.utils import calculate_digest
-from pulpcore.plugin.models import Artifact, ContentArtifact, Content
+from pulpcore.plugin.models import (
+    Artifact,
+    ContentArtifact,
+    Content,
+    PulpTemporaryFile,
+)
 
 
 def get_or_create_blob(layer_json, manifest, path):
@@ -43,7 +48,7 @@ def get_or_create_blob(layer_json, manifest, path):
             artifact=layer_artifact, content=blob, relative_path=layer_json["digest"]
         ).save()
     if layer_json["mediaType"] != MEDIA_TYPE.CONFIG_BLOB_OCI:
-        BlobManifest(manifest=manifest, manifest_blob=blob).save()
+        BlobManifest.objects.get_or_create(manifest=manifest, manifest_blob=blob)
     return blob
 
 
@@ -68,15 +73,13 @@ def add_image_from_directory_to_repository(path, repository, tag):
     manifest_digest = calculate_digest(bytes_data)
     manifest_text_data = bytes_data.decode("utf-8")
 
-    manifest = Manifest(
+    manifest, _ = Manifest.objects.get_or_create(
         digest=manifest_digest,
         schema_version=2,
         media_type=MEDIA_TYPE.MANIFEST_OCI,
         data=manifest_text_data,
     )
-    manifest.save()
-    tag = Tag(name=tag, tagged_manifest=manifest)
-    tag.save()
+    tag, _ = Tag.objects.get_or_create(name=tag, tagged_manifest=manifest)
 
     with repository.new_version() as new_repo_version:
         manifest_json = json.loads(manifest_text_data)
@@ -96,7 +99,11 @@ def add_image_from_directory_to_repository(path, repository, tag):
 
 
 def build_image_from_containerfile(
-    containerfile_pk=None, artifacts=None, repository_pk=None, tag=None
+    containerfile_artifact_pk=None,
+    containerfile_tempfile_pk=None,
+    content_artifact_pks=None,
+    repository_pk=None,
+    tag=None,
 ):
     """
     Builds an OCI container image from a Containerfile.
@@ -105,10 +112,11 @@ def build_image_from_containerfile(
     values. The Containerfile can make use of these files during build process.
 
     Args:
-        containerfile_pk (str): The pk of an Artifact that contains the Containerfile
-        artifacts (dict): A dictionary where each key is an artifact PK and the value is it's
-                          relative path (name) inside the /pulp_working_directory of the build
-                          container executing the Containerfile.
+        containerfile_artifact_pk (str): The pk of an Artifact that contains the Containerfile
+        containerfile_tempfile_pk (str): The pk of a PulpTemporaryFile that contains
+                                         the Containerfile
+        content_artifact_pk (list): The list of pks of ContentArtifacts used in the build context
+                                    of the Containerfile
         repository_pk (str): The pk of a Repository to add the OCI container image
         tag (str): Tag name for the new image in the repository
 
@@ -117,26 +125,33 @@ def build_image_from_containerfile(
         image and tag.
 
     """
-    containerfile = Artifact.objects.get(pk=containerfile_pk)
+    if containerfile_tempfile_pk:
+        containerfile = PulpTemporaryFile.objects.get(pk=containerfile_tempfile_pk)
+    elif containerfile_artifact_pk:
+        containerfile = Artifact.objects.get(pk=containerfile_artifact_pk)
+    else:
+        raise RuntimeError(
+            "Neither an artifact nor temporary file for the Containerfile was specified."
+        )
+
     repository = ContainerRepository.objects.get(pk=repository_pk)
     name = str(uuid4())
     with tempfile.TemporaryDirectory(dir=".") as working_directory:
         working_directory = os.path.abspath(working_directory)
         context_path = os.path.join(working_directory, "context")
         os.makedirs(context_path, exist_ok=True)
-        for key, val in artifacts.items():
-            artifact = Artifact.objects.get(pk=key)
-            dest_path = os.path.join(context_path, val)
-            dirs = os.path.split(dest_path)[0]
-            if dirs:
-                os.makedirs(dirs, exist_ok=True)
-            with open(dest_path, "wb") as dest:
-                shutil.copyfileobj(artifact.file, dest)
-
-            containerfile_path = os.path.join(working_directory, "Containerfile")
 
+        containerfile_path = os.path.join(working_directory, "Containerfile")
         with open(containerfile_path, "wb") as dest:
             shutil.copyfileobj(containerfile.file, dest)
+
+        if content_artifact_pks:
+            content_artifacts = ContentArtifact.objects.filter(pk__in=content_artifact_pks)
+            for content_artifact in content_artifacts.select_related("artifact").iterator():
+                _copy_file_from_artifact(
+                    context_path, content_artifact.relative_path, content_artifact.artifact.file
+                )
+
         bud_cp = subprocess.run(
             [
                 "podman",
@@ -166,3 +181,12 @@ def build_image_from_containerfile(
         repository_version = add_image_from_directory_to_repository(image_dir, repository, tag)
 
     return repository_version
+
+
+def _copy_file_from_artifact(context_path, relative_path, artifact):
+    dest_path = os.path.join(context_path, relative_path)
+    dirs = os.path.dirname(dest_path)
+    if dirs:
+        os.makedirs(dirs, exist_ok=True)
+    with open(dest_path, "wb") as dest:
+        shutil.copyfileobj(artifact.file, dest)