From d069f2afcacd84c81a42fbd32a3926140226aa71 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 24 Jun 2024 09:27:14 -0400 Subject: [PATCH] Add ability to download file for content uploads fixes: #4608 --- CHANGES/plugin_api/4608.feature | 1 + .../functional/api/test_crud_content_unit.py | 36 ++++++++++++ pulpcore/plugin/serializers/content.py | 56 ++++++++++++++++++- 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 CHANGES/plugin_api/4608.feature diff --git a/CHANGES/plugin_api/4608.feature b/CHANGES/plugin_api/4608.feature new file mode 100644 index 0000000000..752ae5480f --- /dev/null +++ b/CHANGES/plugin_api/4608.feature @@ -0,0 +1 @@ +Added new `url` field to UploadSerializerFieldsMixin that will download the file used for the content upload task. diff --git a/pulp_file/tests/functional/api/test_crud_content_unit.py b/pulp_file/tests/functional/api/test_crud_content_unit.py index 00d423b506..b08b52d98b 100644 --- a/pulp_file/tests/functional/api/test_crud_content_unit.py +++ b/pulp_file/tests/functional/api/test_crud_content_unit.py @@ -253,3 +253,39 @@ def test_create_file_content_from_chunked_upload( # Upload gets deleted even though no new content got created with pytest.raises(coreApiException): pulpcore_bindings.UploadsApi.read(upload.pulp_href) + + +@pytest.mark.parallel +def test_create_file_from_url( + file_bindings, + file_repository_factory, + file_remote_factory, + file_distribution_factory, + basic_manifest_path, + monitor_task, +): + # Test create w/ url + remote = file_remote_factory(manifest_path=basic_manifest_path) + body = {"url": remote.url, "relative_path": "PULP_MANIFEST"} + response = file_bindings.ContentFilesApi.create(**body) + task = monitor_task(response.task) + assert len(task.created_resources) == 1 + assert "api/v3/content/file/files/" in task.created_resources[0] + + # Set up + repo1 = file_repository_factory(autopublish=True) + body = {"remote": remote.pulp_href} + monitor_task(file_bindings.RepositoriesFileApi.sync(repo1.pulp_href, body).task) + distro = file_distribution_factory(repository=repo1.pulp_href) + content = file_bindings.ContentFilesApi.list( + repository_version=f"{repo1.versions_href}1/", relative_path="1.iso" + ).results[0] + + # Test create w/ url for already existing content + response = file_bindings.ContentFilesApi.create( + url=f"{distro.base_url}1.iso", + relative_path="1.iso", + ) + task = monitor_task(response.task) + assert len(task.created_resources) == 1 + assert task.created_resources[0] == content.pulp_href diff --git a/pulpcore/plugin/serializers/content.py b/pulpcore/plugin/serializers/content.py index 9b2475a6cc..4a2b32edf2 100644 --- a/pulpcore/plugin/serializers/content.py +++ b/pulpcore/plugin/serializers/content.py @@ -4,12 +4,14 @@ from django.db import DatabaseError from rest_framework.serializers import ( + CharField, FileField, Serializer, ValidationError, ) +from urllib.parse import urlparse from pulpcore.app.files import PulpTemporaryUploadedFile -from pulpcore.app.models import Artifact, PulpTemporaryFile, Upload, UploadChunk +from pulpcore.app.models import Artifact, PulpTemporaryFile, Remote, Upload, UploadChunk from pulpcore.app.serializers import ( RelatedField, ArtifactSerializer, @@ -22,6 +24,8 @@ class UploadSerializerFieldsMixin(Serializer): """A mixin class that contains fields and methods common to content upload serializers.""" + REMOTE_CLASS = Remote + file = FileField( help_text=_("An uploaded file that may be turned into the content unit."), required=False, @@ -34,6 +38,45 @@ class UploadSerializerFieldsMixin(Serializer): view_name=r"uploads-detail", queryset=Upload.objects.all(), ) + url = CharField( + help_text=_("A url that Pulp can download and turn into the content unit."), + required=False, + write_only=True, + ) + + def validate_url(self, value): + """Parse out the auth if provided.""" + url_parse = urlparse(value) + if url_parse.username or url_parse.password: + kwargs = {"username": url_parse.username, "password": url_parse.password} + if self.context.get("remote_kwargs"): + self.context["remote_kwargs"].update(kwargs) + else: + self.context["remote_kwargs"] = kwargs + + return url_parse._replace(netloc=url_parse.netloc.split("@")[-1]).geturl() + + def download(self, url, expected_digests=None, expected_size=None): + """ + Downloads & returns the file from the url. + + Plugins can overwrite this method on their content serializers to get specific download + behavior for their content types. + + Args: + url (str): A url that Pulp can download + expected_digests (dict): A dict of expected digests. + expected_size (int): The expected size in bytes. + + Returns: + PulpTemporaryUploadedFile: the downloaded file + """ + remote = self.REMOTE_CLASS(url=url, **self.context.get("remote_kwargs", {})) + downloader = remote.get_downloader( + url=url, expected_digests=expected_digests, expected_size=expected_size + ) + result = downloader.fetch() + return PulpTemporaryUploadedFile.from_file(open(result.path, "rb")) def validate(self, data): """Validate that we have an Artifact/File or can create one.""" @@ -42,7 +85,9 @@ def validate(self, data): if "request" in self.context: upload_fields = { - field for field in self.Meta.fields if field in {"file", "upload", "artifact"} + field + for field in self.Meta.fields + if field in {"file", "upload", "artifact", "url"} } if len(upload_fields.intersection(data.keys())) != 1: raise ValidationError( @@ -81,6 +126,12 @@ def deferred_validate(self, data): elif pulp_temp_file_pk := self.context.get("pulp_temp_file_pk"): pulp_temp_file = PulpTemporaryFile.objects.get(pk=pulp_temp_file_pk) data["file"] = PulpTemporaryUploadedFile.from_file(pulp_temp_file.file) + elif url := data.pop("url", None): + expected_digests = data.get("expected_digests", None) + expected_size = data.get("expected_size", None) + data["file"] = self.download( + url, expected_digests=expected_digests, expected_size=expected_size + ) return data def create(self, validated_data): @@ -96,6 +147,7 @@ class Meta: fields = ( "file", "upload", + "url", )