diff --git a/pulp_container/app/modelresource.py b/pulp_container/app/modelresource.py index dfd7e8f85..0116db57e 100644 --- a/pulp_container/app/modelresource.py +++ b/pulp_container/app/modelresource.py @@ -4,6 +4,7 @@ from pulp_container.app.models import ( Blob, + ConfigBlob, ContainerRepository, ContainerPushRepository, Manifest, @@ -60,6 +61,22 @@ class Meta: exclude = RepositoryResource.Meta.exclude + ("manifest_signing_service",) +class ConfigBlobResource(BaseContentResource): + """ + Resource for import/export of configblob entities + """ + + def set_up_queryset(self): + """ + :return: ConfigBlobs specific to a specified repo-version. + """ + return ConfigBlob.objects.filter(pk__in=self.repo_version.content).order_by("content_ptr_id") + + class Meta: + model = ConfigBlob + import_id_fields = model.natural_key_fields() + + class BlobResource(BaseContentResource): """ Resource for import/export of blob entities @@ -89,7 +106,7 @@ class ManifestResource(BaseContentResource): config_blob = fields.Field( column_name="config_blob", attribute="config_blob", - widget=widgets.ForeignKeyWidget(Blob, field="digest"), + widget=widgets.ForeignKeyWidget(ConfigBlob, field="digest"), ) def set_up_queryset(self): @@ -179,6 +196,7 @@ class Meta: IMPORT_ORDER = [ + ConfigBlobResource, BlobResource, ManifestResource, ManifestListManifestResource, diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index b97ce15b1..e2833c557 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -351,14 +351,13 @@ async def handle_blobs(self, manifest_dc, content_data): for layer in content_data.get("layers") or content_data.get("fsLayers"): if not self._include_layer(layer): continue - blob_dc, _ = await self.create_blob(layer) + blob_dc = self.create_blob(layer) manifest_dc.extra_data["blob_dcs"].append(blob_dc) await self.put(blob_dc) layer = content_data.get("config", None) if layer: - blob_dc, config_blob_dc = await self.create_blob(layer, deferred_download=False) + config_blob_dc = await self.create_config_blob(layer) manifest_dc.extra_data["config_blob_dc"] = config_blob_dc - await self.put(blob_dc) await self.put(config_blob_dc) def create_tagged_manifest_list(self, tag_name, saved_artifact, manifest_list_data, media_type): @@ -517,36 +516,37 @@ async def create_listed_manifest(self, manifest_data): ) return {"manifest_dc": man_dc, "platform": platform, "content_data": content_data} - async def create_config_blob(self, blob_data, blob_url, digest, blob): - config_blob_dc = None - mediaType = blob_data.get("mediaType") - - if mediaType in [MEDIA_TYPE.CONFIG_BLOB, MEDIA_TYPE.CONFIG_BLOB_OCI]: - downloader = self.remote.get_downloader(url=blob_url) - response = await downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS}) - with open(response.path, "rb") as content_file: - raw_data = content_file.read() - content_data = json.loads(raw_data) - config_blob = ConfigBlob( - data = raw_data.decode("utf-8"), - digest = digest, - architecture=content_data.get("architecture"), - os=content_data.get("os"), - created=content_data.get("created"), - author=content_data.get("author", ""), - os_version=content_data.get("os_version", ""), - os_features=content_data.get("os_features", ""), - variant=content_data.get("variant", ""), - rootfs=content_data.get("rootfs", {}), - config=content_data.get("config", {}), - history=content_data.get("history", {}), - ) - blob = Blob(digest=digest) - config_blob_dc = DeclarativeContent(content=config_blob) + async def create_config_blob(self, blob_data): + digest = blob_data.get("digest") or blob_data.get("blobSum") + relative_url = "/v2/{name}/blobs/{digest}".format( + name=self.remote.namespaced_upstream_name, digest=digest + ) + blob_url = urljoin(self.remote.url, relative_url) + + downloader = self.remote.get_downloader(url=blob_url) + response = await downloader.run(extra_data={"headers": V2_ACCEPT_HEADERS}) + with open(response.path, "rb") as content_file: + raw_data = content_file.read() + content_data = json.loads(raw_data) + config_blob = ConfigBlob( + data = raw_data.decode("utf-8"), + digest = digest, + architecture=content_data.get("architecture"), + os=content_data.get("os"), + created=content_data.get("created"), + author=content_data.get("author", ""), + os_version=content_data.get("os_version", ""), + os_features=content_data.get("os_features", ""), + variant=content_data.get("variant", ""), + rootfs=content_data.get("rootfs", {}), + config=content_data.get("config", {}), + history=content_data.get("history", {}), + ) + config_blob_dc = DeclarativeContent(content=config_blob) - return blob, config_blob_dc + return config_blob_dc - async def create_blob(self, blob_data, deferred_download=True): + def create_blob(self, blob_data, deferred_download=True): """ Create blob. @@ -571,10 +571,9 @@ async def create_blob(self, blob_data, deferred_download=True): deferred_download=deferred_download and self.deferred_download, ) - blob, config_blob_dc = await self.create_config_blob(blob_data, blob_url, digest, blob) blob_dc = DeclarativeContent(content=blob, d_artifacts=[da]) - return blob_dc, config_blob_dc + return blob_dc async def create_signatures(self, man_dc, signature_source): """