draft/wip

pulp_container/app/management/commands/container-handle-image-data.py

@@ -85,20 +85,7 @@ def update_config_blob(self, manifest):
         digest = manifest.config_blob.digest
         with transaction.atomic():
             try:
-                blob = ConfigBlob(
-                    data=raw_manifest,
-                    digest=digest,
-                    architecture=config_blob.get("architecture"),
-                    os=config_blob.get("os"),
-                    created=config_blob.get("created"),
-                    author=config_blob.get("author", ""),
-                    os_version=config_blob.get("os_version", ""),
-                    os_features=config_blob.get("os_features", ""),
-                    variant=config_blob.get("variant", ""),
-                    rootfs=config_blob.get("rootfs", {}),
-                    config=config_blob.get("config", {}),
-                    history=config_blob.get("history", {}),
-                )
+                blob = ConfigBlob.build(raw_manifest,digest,config_blob)
                 blob.save()
             except IntegrityError:
                 blob = ConfigBlob.objects.get(digest=digest)

pulp_container/app/migrations/0039_artifactless_config_blob.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.10 on 2024-04-03 10:54
+# Generated by Django 4.2.10 on 2024-04-03 17:18
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -11,27 +11,17 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.AlterField(
-            model_name='manifest',
-            name='config_blob',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config', to='container.blob'),
-        ),
         migrations.CreateModel(
             name='ConfigBlob',
             fields=[
                 ('content_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='core.content')),
                 ('data', models.TextField()),
-                ('architecture', models.TextField()),
-                ('os', models.TextField()),
-                ('rootfs', models.JSONField(default=dict)),
-                ('digest', models.TextField(db_index=True)),
                 ('created', models.DateTimeField()),
                 ('author', models.TextField(blank=True, default='')),
-                ('os_version', models.TextField(blank=True, default='')),
-                ('os_features', models.TextField(blank=True, default='')),
-                ('variant', models.TextField(blank=True, default='')),
+                ('os', models.TextField()),
+                ('architecture', models.TextField()),
+                ('digest', models.TextField(db_index=True)),
                 ('config', models.JSONField(default=dict)),
-                ('history', models.JSONField(default=dict)),
             ],
             options={
                 'default_related_name': '%(app_label)s_%(model_name)s',
@@ -52,6 +42,6 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='manifest',
             name='config',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='config_blob', to='container.configblob'),
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='manifest_config_blob', to='container.configblob'),
         ),
     ]

pulp_container/app/models.py

@@ -61,20 +61,25 @@ class ConfigBlob(Content):
     TYPE = "config-blob"
 
     data = models.TextField()
-    architecture = models.TextField()
-    os = models.TextField()
-    rootfs = models.JSONField(default=dict)
-    digest = models.TextField(db_index=True)
-
     created = models.DateTimeField()
     author = models.TextField(default="", blank=True)
-    os_version = models.TextField(default="", blank=True)
-    os_features = models.TextField(default="", blank=True)
-    variant = models.TextField(default="", blank=True)
-
+    os = models.TextField()
+    architecture = models.TextField()
+    digest = models.TextField(db_index=True)
     config = models.JSONField(default=dict)
-    history = models.JSONField(default=dict)
 
+    @classmethod
+    def build(cls,raw_data,digest,**content_data):
+        config_blob = cls(
+            data=raw_data,
+            digest=digest,
+            architecture=content_data.get("architecture"),
+            os=content_data.get("os"),
+            created=content_data.get("created"),
+            author=content_data.get("author", ""),
+            config=content_data.get("config", {}),
+        )
+        return config_blob
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("digest",)
@@ -149,10 +154,10 @@ class Manifest(Content):
 
     blobs = models.ManyToManyField(Blob, through="BlobManifest")
     config_blob = models.ForeignKey(
-        Blob, related_name="config", null=True, on_delete=models.CASCADE
+        Blob, related_name="config_blob", null=True, on_delete=models.CASCADE
     )
     config = models.ForeignKey(
-        ConfigBlob, related_name="config_blob", null=True, on_delete=models.CASCADE
+        ConfigBlob, related_name="manifest_config_blob", null=True, on_delete=models.CASCADE
     )
 
     # Order matters for through fields, (source, target)

pulp_container/app/registry.py

@@ -1,5 +1,4 @@
 import json
-import time
 import logging
 import os
 
@@ -293,7 +292,6 @@ async def get_by_digest(self, request):
                 # "/pulp/container/{path:.+}/{content:(blobs|manifests)}/sha256:{digest:.+}"
                 content_type = request.match_info["content"]
                 if content_type == "manifests":
-                    time.sleep(10)
                     raw_manifest, digest, media_type = await pull_downloader.download_manifest()
                     headers = {
                         "Content-Type": media_type,
@@ -431,12 +429,11 @@ async def run_pipeline(self, saved_artifact):
                 "tag_name": self.identifier,
             },
         )
-        time.sleep(5)
 
     async def init_pending_content(self, digest, manifest_data, media_type, artifact):
         if config := manifest_data.get("config", None):
             config_digest = config["digest"]
-            config_blob = await ConfigBlob.objects.aget(digest=config_digest)
+            config_blob = await self.save_config_blob(config_digest)
             await sync_to_async(self.repository.pending_config_blobs.add)(config_blob)
         else:
             config_blob = None
@@ -505,3 +502,23 @@ async def save_blob(self, digest, manifest):
             await ra.asave()
 
         return blob
+
+    async def save_config_blob(self, config_digest):
+        blob_relative_url = "/v2/{name}/blobs/{digest}".format(
+            name=self.remote.namespaced_upstream_name, digest=config_digest
+        )
+        blob_url = urljoin(self.remote.url, blob_relative_url)
+        downloader = self.remote.get_downloader(url=blob_url)
+        response = await downloader.run()
+
+        with open(response.path, "r") as content_file:
+            raw_data = content_file.read()
+        content_data = json.loads(raw_data)
+        config_blob = ConfigBlob(raw_data,config_digest,content_data)
+
+        try:
+            await config_blob.asave()
+        except IntegrityError:
+            config_blob = await ConfigBlob.objects.aget(digest=config_digest)
+            await sync_to_async(config_blob.touch)()
+        return config_blob

pulp_container/app/registry_api.py

@@ -874,20 +874,7 @@ def partial_update(self, request, path, pk=None):
     def create_config_blob(self, manifest, raw_manifest, digest):
         with transaction.atomic():
             try:
-                blob = models.ConfigBlob(
-                    data=raw_manifest,
-                    digest=digest,
-                    architecture=manifest.get("architecture"),
-                    os=manifest.get("os"),
-                    created=manifest.get("created"),
-                    author=manifest.get("author", ""),
-                    os_version=manifest.get("os_version", ""),
-                    os_features=manifest.get("os_features", ""),
-                    variant=manifest.get("variant", ""),
-                    rootfs=manifest.get("rootfs", {}),
-                    config=manifest.get("config", {}),
-                    history=manifest.get("history", {}),
-                )
+                blob = models.ConfigBlob.build(raw_manifest,digest,manifest)
                 blob.save()
             except IntegrityError:
                 blob = models.ConfigBlob.objects.get(digest=digest)

pulp_container/app/tasks/builder.py

@@ -41,20 +41,7 @@ def get_or_create_blob(layer_json, manifest, path):
             with open(layer_file_name, "r") as content_file:
                 raw_data = content_file.read()
             content_data = json.loads(raw_data)
-            config_blob = ConfigBlob(
-                data=raw_data,
-                digest=layer_json["digest"],
-                architecture=content_data.get("architecture"),
-                os=content_data.get("os"),
-                created=content_data.get("created"),
-                author=content_data.get("author", ""),
-                os_version=content_data.get("os_version", ""),
-                os_features=content_data.get("os_features", ""),
-                variant=content_data.get("variant", ""),
-                rootfs=content_data.get("rootfs", {}),
-                config=content_data.get("config", {}),
-                history=content_data.get("history", {}),
-            )
+            config_blob = ConfigBlob.build(raw_data,layer_json["digest"],content_data)
             config_blob.save()
             return config_blob
         else:

pulp_container/app/tasks/sync_stages.py

@@ -528,20 +528,7 @@ async def create_config_blob(self, blob_data):
         with open(response.path, "rb") as content_file:
             raw_data = content_file.read()
         content_data = json.loads(raw_data)
-        config_blob = ConfigBlob(
-            data=raw_data.decode("utf-8"),
-            digest=digest,
-            architecture=content_data.get("architecture"),
-            os=content_data.get("os"),
-            created=content_data.get("created"),
-            author=content_data.get("author", ""),
-            os_version=content_data.get("os_version", ""),
-            os_features=content_data.get("os_features", ""),
-            variant=content_data.get("variant", ""),
-            rootfs=content_data.get("rootfs", {}),
-            config=content_data.get("config", {}),
-            history=content_data.get("history", {}),
-        )
+        config_blob = ConfigBlob.build(raw_data.decode("utf-8"),digest,content_data)
         config_blob_dc = DeclarativeContent(content=config_blob)
 
         return config_blob_dc