diff --git a/pulp_container/app/management/commands/container-handle-image-data.py b/pulp_container/app/management/commands/container-handle-image-data.py index 1f70c1323..ad26ff525 100644 --- a/pulp_container/app/management/commands/container-handle-image-data.py +++ b/pulp_container/app/management/commands/container-handle-image-data.py @@ -85,20 +85,7 @@ def update_config_blob(self, manifest): digest = manifest.config_blob.digest with transaction.atomic(): try: - blob = ConfigBlob( - data=raw_manifest, - digest=digest, - architecture=config_blob.get("architecture"), - os=config_blob.get("os"), - created=config_blob.get("created"), - author=config_blob.get("author", ""), - os_version=config_blob.get("os_version", ""), - os_features=config_blob.get("os_features", ""), - variant=config_blob.get("variant", ""), - rootfs=config_blob.get("rootfs", {}), - config=config_blob.get("config", {}), - history=config_blob.get("history", {}), - ) + blob = ConfigBlob.build(raw_manifest,digest,config_blob) blob.save() except IntegrityError: blob = ConfigBlob.objects.get(digest=digest) diff --git a/pulp_container/app/migrations/0039_artifactless_config_blob.py b/pulp_container/app/migrations/0039_artifactless_config_blob.py index 318916f92..05fbca3a4 100644 --- a/pulp_container/app/migrations/0039_artifactless_config_blob.py +++ b/pulp_container/app/migrations/0039_artifactless_config_blob.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-04-03 10:54 +# Generated by Django 4.2.10 on 2024-04-03 17:18 from django.db import migrations, models import django.db.models.deletion @@ -11,27 +11,17 @@ class Migration(migrations.Migration): ] operations = [ - migrations.AlterField( - model_name='manifest', - name='config_blob', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config', to='container.blob'), - ), migrations.CreateModel( name='ConfigBlob', fields=[ ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content')), ('data', models.TextField()), - ('architecture', models.TextField()), - ('os', models.TextField()), - ('rootfs', models.JSONField(default=dict)), - ('digest', models.TextField(db_index=True)), ('created', models.DateTimeField()), ('author', models.TextField(blank=True, default='')), - ('os_version', models.TextField(blank=True, default='')), - ('os_features', models.TextField(blank=True, default='')), - ('variant', models.TextField(blank=True, default='')), + ('os', models.TextField()), + ('architecture', models.TextField()), + ('digest', models.TextField(db_index=True)), ('config', models.JSONField(default=dict)), - ('history', models.JSONField(default=dict)), ], options={ 'default_related_name': '%(app_label)s_%(model_name)s', @@ -52,6 +42,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name='manifest', name='config', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config_blob', to='container.configblob'), + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='manifest_config_blob', to='container.configblob'), ), ] diff --git a/pulp_container/app/models.py b/pulp_container/app/models.py index 6835792e6..5ebfc93c6 100644 --- a/pulp_container/app/models.py +++ b/pulp_container/app/models.py @@ -61,20 +61,25 @@ class ConfigBlob(Content): TYPE = "config-blob" data = models.TextField() - architecture = models.TextField() - os = models.TextField() - rootfs = models.JSONField(default=dict) - digest = models.TextField(db_index=True) - created = models.DateTimeField() author = models.TextField(default="", blank=True) - os_version = models.TextField(default="", blank=True) - os_features = models.TextField(default="", blank=True) - variant = models.TextField(default="", blank=True) - + os = models.TextField() + architecture = models.TextField() + digest = models.TextField(db_index=True) config = models.JSONField(default=dict) - history = models.JSONField(default=dict) + @classmethod + def build(cls,raw_data,digest,**content_data): + config_blob = cls( + data=raw_data, + digest=digest, + architecture=content_data.get("architecture"), + os=content_data.get("os"), + created=content_data.get("created"), + author=content_data.get("author", ""), + config=content_data.get("config", {}), + ) + return config_blob class Meta: default_related_name = "%(app_label)s_%(model_name)s" unique_together = ("digest",) @@ -149,10 +154,10 @@ class Manifest(Content): blobs = models.ManyToManyField(Blob, through="BlobManifest") config_blob = models.ForeignKey( - Blob, related_name="config", null=True, on_delete=models.CASCADE + Blob, related_name="config_blob", null=True, on_delete=models.CASCADE ) config = models.ForeignKey( - ConfigBlob, related_name="config_blob", null=True, on_delete=models.CASCADE + ConfigBlob, related_name="manifest_config_blob", null=True, on_delete=models.CASCADE ) # Order matters for through fields, (source, target) diff --git a/pulp_container/app/registry.py b/pulp_container/app/registry.py index 18996fc80..af315080c 100644 --- a/pulp_container/app/registry.py +++ b/pulp_container/app/registry.py @@ -1,5 +1,4 @@ import json -import time import logging import os @@ -293,7 +292,6 @@ async def get_by_digest(self, request): # "/pulp/container/{path:.+}/{content:(blobs|manifests)}/sha256:{digest:.+}" content_type = request.match_info["content"] if content_type == "manifests": - time.sleep(10) raw_manifest, digest, media_type = await pull_downloader.download_manifest() headers = { "Content-Type": media_type, @@ -431,12 +429,11 @@ async def run_pipeline(self, saved_artifact): "tag_name": self.identifier, }, ) - time.sleep(5) async def init_pending_content(self, digest, manifest_data, media_type, artifact): if config := manifest_data.get("config", None): config_digest = config["digest"] - config_blob = await ConfigBlob.objects.aget(digest=config_digest) + config_blob = await self.save_config_blob(config_digest) await sync_to_async(self.repository.pending_config_blobs.add)(config_blob) else: config_blob = None @@ -505,3 +502,23 @@ async def save_blob(self, digest, manifest): await ra.asave() return blob + + async def save_config_blob(self, config_digest): + blob_relative_url = "/v2/{name}/blobs/{digest}".format( + name=self.remote.namespaced_upstream_name, digest=config_digest + ) + blob_url = urljoin(self.remote.url, blob_relative_url) + downloader = self.remote.get_downloader(url=blob_url) + response = await downloader.run() + + with open(response.path, "r") as content_file: + raw_data = content_file.read() + content_data = json.loads(raw_data) + config_blob = ConfigBlob(raw_data,config_digest,content_data) + + try: + await config_blob.asave() + except IntegrityError: + config_blob = await ConfigBlob.objects.aget(digest=config_digest) + await sync_to_async(config_blob.touch)() + return config_blob diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 94f085d68..271db25d5 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -874,20 +874,7 @@ def partial_update(self, request, path, pk=None): def create_config_blob(self, manifest, raw_manifest, digest): with transaction.atomic(): try: - blob = models.ConfigBlob( - data=raw_manifest, - digest=digest, - architecture=manifest.get("architecture"), - os=manifest.get("os"), - created=manifest.get("created"), - author=manifest.get("author", ""), - os_version=manifest.get("os_version", ""), - os_features=manifest.get("os_features", ""), - variant=manifest.get("variant", ""), - rootfs=manifest.get("rootfs", {}), - config=manifest.get("config", {}), - history=manifest.get("history", {}), - ) + blob = models.ConfigBlob.build(raw_manifest,digest,manifest) blob.save() except IntegrityError: blob = models.ConfigBlob.objects.get(digest=digest) diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index f11215e7b..408631a41 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -41,20 +41,7 @@ def get_or_create_blob(layer_json, manifest, path): with open(layer_file_name, "r") as content_file: raw_data = content_file.read() content_data = json.loads(raw_data) - config_blob = ConfigBlob( - data=raw_data, - digest=layer_json["digest"], - architecture=content_data.get("architecture"), - os=content_data.get("os"), - created=content_data.get("created"), - author=content_data.get("author", ""), - os_version=content_data.get("os_version", ""), - os_features=content_data.get("os_features", ""), - variant=content_data.get("variant", ""), - rootfs=content_data.get("rootfs", {}), - config=content_data.get("config", {}), - history=content_data.get("history", {}), - ) + config_blob = ConfigBlob.build(raw_data,layer_json["digest"],content_data) config_blob.save() return config_blob else: diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 8367cb6ea..af8a3abd8 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -528,20 +528,7 @@ async def create_config_blob(self, blob_data): with open(response.path, "rb") as content_file: raw_data = content_file.read() content_data = json.loads(raw_data) - config_blob = ConfigBlob( - data=raw_data.decode("utf-8"), - digest=digest, - architecture=content_data.get("architecture"), - os=content_data.get("os"), - created=content_data.get("created"), - author=content_data.get("author", ""), - os_version=content_data.get("os_version", ""), - os_features=content_data.get("os_features", ""), - variant=content_data.get("variant", ""), - rootfs=content_data.get("rootfs", {}), - config=content_data.get("config", {}), - history=content_data.get("history", {}), - ) + config_blob = ConfigBlob.build(raw_data.decode("utf-8"),digest,content_data) config_blob_dc = DeclarativeContent(content=config_blob) return config_blob_dc