Add ability to download file for content uploads

fixes: #4608

CHANGES/plugin_api/4608.feature

@@ -0,0 +1 @@
+Added new `url` field to UploadSerializerFieldsMixin that will download the file used for the content upload task.

pulp_file/tests/functional/api/test_crud_content_unit.py

@@ -253,3 +253,39 @@ def test_create_file_content_from_chunked_upload(
     # Upload gets deleted even though no new content got created
     with pytest.raises(coreApiException):
         pulpcore_bindings.UploadsApi.read(upload.pulp_href)
+
+
+@pytest.mark.parallel
+def test_create_file_from_url(
+    file_bindings,
+    file_repository_factory,
+    file_remote_factory,
+    file_distribution_factory,
+    basic_manifest_path,
+    monitor_task,
+):
+    # Test create w/ url
+    remote = file_remote_factory(manifest_path=basic_manifest_path)
+    body = {"url": remote.url, "relative_path": "PULP_MANIFEST"}
+    response = file_bindings.ContentFilesApi.create(**body)
+    task = monitor_task(response.task)
+    assert len(task.created_resources) == 1
+    assert "api/v3/content/file/files/" in task.created_resources[0]
+
+    # Set up
+    repo1 = file_repository_factory(autopublish=True)
+    body = {"remote": remote.pulp_href}
+    monitor_task(file_bindings.RepositoriesFileApi.sync(repo1.pulp_href, body).task)
+    distro = file_distribution_factory(repository=repo1.pulp_href)
+    content = file_bindings.ContentFilesApi.list(
+        repository_version=f"{repo1.versions_href}1/", relative_path="1.iso"
+    ).results[0]
+
+    # Test create w/ url for already existing content
+    response = file_bindings.ContentFilesApi.create(
+        url=f"{distro.base_url}1.iso",
+        relative_path="1.iso",
+    )
+    task = monitor_task(response.task)
+    assert len(task.created_resources) == 1
+    assert task.created_resources[0] == content.pulp_href

pulpcore/plugin/serializers/content.py

@@ -4,12 +4,14 @@
 
 from django.db import DatabaseError
 from rest_framework.serializers import (
+    CharField,
     FileField,
     Serializer,
     ValidationError,
 )
+from urllib.parse import urlparse
 from pulpcore.app.files import PulpTemporaryUploadedFile
-from pulpcore.app.models import Artifact, PulpTemporaryFile, Upload, UploadChunk
+from pulpcore.app.models import Artifact, PulpTemporaryFile, Remote, Upload, UploadChunk
 from pulpcore.app.serializers import (
     RelatedField,
     ArtifactSerializer,
@@ -22,6 +24,8 @@
 class UploadSerializerFieldsMixin(Serializer):
     """A mixin class that contains fields and methods common to content upload serializers."""
 
+    REMOTE_CLASS = Remote
+
     file = FileField(
         help_text=_("An uploaded file that may be turned into the content unit."),
         required=False,
@@ -34,6 +38,45 @@ class UploadSerializerFieldsMixin(Serializer):
         view_name=r"uploads-detail",
         queryset=Upload.objects.all(),
     )
+    url = CharField(
+        help_text=_("A url that Pulp can download and turn into the content unit."),
+        required=False,
+        write_only=True,
+    )
+
+    def validate_url(self, value):
+        """Parse out the auth if provided."""
+        url_parse = urlparse(value)
+        if url_parse.username or url_parse.password:
+            kwargs = {"username": url_parse.username, "password": url_parse.password}
+            if self.context.get("remote_kwargs"):
+                self.context["remote_kwargs"].update(kwargs)
+            else:
+                self.context["remote_kwargs"] = kwargs
+
+        return url_parse._replace(netloc=url_parse.netloc.split("@")[-1]).geturl()
+
+    def download(self, url, expected_digests=None, expected_size=None):
+        """
+        Downloads & returns the file from the url.
+
+        Plugins can overwrite this method on their content serializers to get specific download
+        behavior for their content types.
+
+        Args:
+            url (str): A url that Pulp can download
+            expected_digests (dict): A dict of expected digests.
+            expected_size (int): The expected size in bytes.
+
+        Returns:
+            PulpTemporaryUploadedFile: the downloaded file
+        """
+        remote = self.REMOTE_CLASS(url=url, **self.context.get("remote_kwargs", {}))
+        downloader = remote.get_downloader(
+            url=url, expected_digests=expected_digests, expected_size=expected_size
+        )
+        result = downloader.fetch()
+        return PulpTemporaryUploadedFile.from_file(open(result.path, "rb"))
 
     def validate(self, data):
         """Validate that we have an Artifact/File or can create one."""
@@ -42,7 +85,9 @@ def validate(self, data):
 
         if "request" in self.context:
             upload_fields = {
-                field for field in self.Meta.fields if field in {"file", "upload", "artifact"}
+                field
+                for field in self.Meta.fields
+                if field in {"file", "upload", "artifact", "url"}
             }
             if len(upload_fields.intersection(data.keys())) != 1:
                 raise ValidationError(
@@ -81,6 +126,12 @@ def deferred_validate(self, data):
         elif pulp_temp_file_pk := self.context.get("pulp_temp_file_pk"):
             pulp_temp_file = PulpTemporaryFile.objects.get(pk=pulp_temp_file_pk)
             data["file"] = PulpTemporaryUploadedFile.from_file(pulp_temp_file.file)
+        elif url := data.pop("url", None):
+            expected_digests = data.get("expected_digests", None)
+            expected_size = data.get("expected_size", None)
+            data["file"] = self.download(
+                url, expected_digests=expected_digests, expected_size=expected_size
+            )
         return data
 
     def create(self, validated_data):
@@ -96,6 +147,7 @@ class Meta:
         fields = (
             "file",
             "upload",
+            "url",
         )