[draft/wip] image push

pulp_container/app/__init__.py

@@ -1,8 +1,11 @@
 from pulpcore.plugin import PulpPluginAppConfig
 
 #import debugpy
-#debugpy.listen(('0.0.0.0',5678))
-#debugpy.wait_for_client()
+#try:
+#    debugpy.listen(('0.0.0.0',5678))
+#    debugpy.wait_for_client()
+#except:
+#    pass
 
 class PulpContainerPluginAppConfig(PulpPluginAppConfig):
     """Entry point for the container plugin."""

pulp_container/app/migrations/0039_configblob_alter_manifest_config_blob_and_more.py → pulp_container/app/migrations/0039_configblob_and_more.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.10 on 2024-03-19 15:16
+# Generated by Django 4.2.10 on 2024-03-21 19:41
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -34,6 +34,16 @@ class Migration(migrations.Migration):
             },
             bases=('core.content',),
         ),
+        migrations.AddField(
+            model_name='containerpushrepository',
+            name='pending_config_blobs',
+            field=models.ManyToManyField(to='container.configblob'),
+        ),
+        migrations.AddField(
+            model_name='containerrepository',
+            name='pending_config_blobs',
+            field=models.ManyToManyField(to='container.configblob'),
+        ),
         migrations.AlterField(
             model_name='manifest',
             name='config_blob',

pulp_container/app/models.py

@@ -180,12 +180,14 @@ def init_annotations(self, manifest_data=None):
         return bool(self.annotations)
 
     def init_labels(self):
-        if self.config_blob:
+        if self.config_blob and "Labels" in self.config_blob.config.keys():
             #if isinstance(self.config_blob.config["Labels"],str):
-            #    self.Labels = json.loads(self.config_blob.config)["Labels"]
+            #    self.labels = json.loads(self.config_blob.config)["Labels"]
             #else:
-            #    self.Labels = self.config_blob.config["Labels"] or {}
-            self.Labels = self.config_blob.config["Labels"] or {}
+            #    self.labels = self.config_blob.config["Labels"] or {}
+            self.labels = self.config_blob.config["Labels"] or {}
+        else:
+            self.labels = {}
 
         return bool(self.labels)
 
@@ -645,6 +647,7 @@ class ContainerRepository(
         ManifestSigningService, on_delete=models.SET_NULL, null=True
     )
     pending_blobs = models.ManyToManyField(Blob)
+    pending_config_blobs = models.ManyToManyField(ConfigBlob)
     pending_manifests = models.ManyToManyField(Manifest)
 
     class Meta:
@@ -702,6 +705,7 @@ class ContainerPushRepository(Repository, AutoAddObjPermsMixin):
         ManifestSigningService, on_delete=models.SET_NULL, null=True
     )
     pending_blobs = models.ManyToManyField(Blob)
+    pending_config_blobs = models.ManyToManyField(ConfigBlob)
     pending_manifests = models.ManyToManyField(Manifest)
 
     class Meta:

pulp_container/app/registry.py

@@ -246,9 +246,6 @@ async def get_by_digest(self, request):
         """
         Return a response to the "GET" action.
         """
-        #import debugpy
-        #debugpy.listen(('0.0.0.0',5679))
-        #debugpy.wait_for_client()
         path = request.match_info["path"]
         digest = "sha256:{digest}".format(digest=request.match_info["digest"])
         distribution = await sync_to_async(self._match_distribution)(path, add_trailing_slash=False)
@@ -261,6 +258,8 @@ async def get_by_digest(self, request):
 
         repository = await repository_version.repository.acast()
         pending_blobs = repository.pending_blobs.values_list("pk")
+        pending_config_blobs = repository.pending_config_blobs.values_list("pk")
+        pending_blobs.union(pending_config_blobs)
         pending_manifests = repository.pending_manifests.values_list("pk")
         pending_content = pending_blobs.union(pending_manifests)
         content = repository_version.content | Content.objects.filter(pk__in=pending_content)
@@ -424,8 +423,9 @@ async def run_pipeline(self, saved_artifact):
     async def init_pending_content(self, digest, manifest_data, media_type, artifact):
         if config := manifest_data.get("config", None):
             config_digest = config["digest"]
+
             config_blob = await self.save_config_blob(config_digest)
-            await sync_to_async(self.repository.pending_blobs.add)(config_blob)
+            await sync_to_async(self.repository.pending_config_blobs.add)(config_blob)
         else:
             config_blob = None
 
@@ -495,29 +495,44 @@ async def save_blob(self, digest, manifest):
         return blob
 
     async def save_config_blob(self, config_digest):
-        blob_relative_url = "/v2/{name}/blobs/{digest}".format(
-            name=self.remote.namespaced_upstream_name, digest=config_digest
-        )
-        blob_url = urljoin(self.remote.url, blob_relative_url)
-        downloader = self.remote.get_downloader(url=blob_url)
-        response = await downloader.run()
-
-        response.artifact_attributes["file"] = response.path
-        saved_artifact = await save_artifact(response.artifact_attributes)
-
-        config_blob = Blob(digest=config_digest)
-        try:
-            await config_blob.asave()
-        except IntegrityError:
-            config_blob = await Blob.objects.aget(digest=config_digest)
-            await sync_to_async(config_blob.touch)()
-
-        content_artifact = ContentArtifact(
-            content=config_blob,
-            artifact=saved_artifact,
-            relative_path=config_digest,
-        )
-        with suppress(IntegrityError):
-            await content_artifact.asave()
-
-        return config_blob
+        return await ConfigBlob.objects.aget(digest=config_digest)
+
+#        blob_relative_url = "/v2/{name}/blobs/{digest}".format(
+#            name=self.remote.namespaced_upstream_name, digest=config_digest
+#        )
+#        blob_url = urljoin(self.remote.url, blob_relative_url)
+#        downloader = self.remote.get_downloader(url=blob_url)
+#        response = await downloader.run()
+#
+#        response.artifact_attributes["file"] = response.path
+#        #saved_artifact = await save_artifact(response.artifact_attributes)
+#
+#        config_blob = ConfigBlob(
+#            data = raw_data.decode("utf-8"),
+#            digest = digest,
+#            architecture=content_data.get("architecture"),
+#            os=content_data.get("os"),
+#            created=content_data.get("created"),
+#            author=content_data.get("author", ""),
+#            os_version=content_data.get("os_version", ""),
+#            os_features=content_data.get("os_features", ""),
+#            variant=content_data.get("variant", ""),
+#            rootfs=content_data.get("rootfs", {}),
+#            config=content_data.get("config", {}),
+#            history=content_data.get("history", {}),
+#        )
+#        try:
+#            await config_blob.asave()
+#        except IntegrityError:
+#            config_blob = await ConfigBlob.objects.aget(digest=config_digest)
+#            await sync_to_async(config_blob.touch)()
+#
+#        #content_artifact = ContentArtifact(
+#        #    content=config_blob,
+#        #    artifact=saved_artifact,
+#        #    relative_path=config_digest,
+#        #)
+#        #with suppress(IntegrityError):
+#        #    await content_artifact.asave()
+#
+#        return config_blob

pulp_container/app/registry_api.py

@@ -780,10 +780,6 @@ def create_single_chunk_artifact(self, chunk):
                 artifact.touch()
         return artifact
 
-    ### CHECK IF THIS WILL NEED TO BE MODIFIED
-    ### CHECK IF THIS WILL NEED TO BE MODIFIED
-    ### CHECK IF THIS WILL NEED TO BE MODIFIED
-    ### CHECK IF THIS WILL NEED TO BE MODIFIED
     def create_blob(self, artifact, digest):
         with transaction.atomic():
             try:
@@ -881,6 +877,38 @@ def partial_update(self, request, path, pk=None):
 
         return UploadResponse(upload=upload, path=path, request=request, status=204)
 
+
+    def is_config_blob(self,blob):
+        required_fields = { "os", "rootfs", "architecture"}
+        if all(config_key in required_fields for config_key in blob):
+            return True
+        return False
+
+
+    def create_config_blob(self, manifest,raw_manifest, digest):
+        with transaction.atomic():
+            try:
+                blob = models.ConfigBlob(
+                    data = raw_manifest,
+                    digest = digest,
+                    architecture=manifest.get("architecture"),
+                    os=manifest.get("os"),
+                    created=manifest.get("created"),
+                    author=manifest.get("author", ""),
+                    os_version=manifest.get("os_version", ""),
+                    os_features=manifest.get("os_features", ""),
+                    variant=manifest.get("variant", ""),
+                    rootfs=manifest.get("rootfs", {}),
+                    config=manifest.get("config", {}),
+                    history=manifest.get("history", {}),
+                )
+                blob.save()
+            except IntegrityError:
+                blob = models.ConfigBlob.objects.get(digest=digest)
+                blob.touch()
+        return blob
+
+
     def put(self, request, path, pk=None):
         """
         Create a blob from uploaded chunks.
@@ -926,10 +954,18 @@ def put(self, request, path, pk=None):
                 artifact = Artifact.objects.get(sha256=artifact.sha256)
                 artifact.touch()
 
-        blob = self.create_blob(artifact, digest)
+        blob = None
+        try:
+            raw_manifest = artifact.file.read().decode("utf-8")
+            manifest = json.loads(raw_manifest)
+            blob = self.create_config_blob(manifest, raw_manifest, digest)
+            repository.pending_config_blobs.add(blob)
+        except UnicodeDecodeError:
+            blob = self.create_blob(artifact, digest)
+            repository.pending_blobs.add(blob)
         upload.delete()
 
-        repository.pending_blobs.add(blob)
+
         return BlobResponse(blob, path, 201, request)
 
 
@@ -992,18 +1028,18 @@ def handle_safe_method(self, request, path, pk):
                 blob = models.ConfigBlob.objects.get(digest=pk)
                 return redirects.redirect_to_content_app("config-blobs",pk)
             except:
-                pass
+                pass ###### WIP PENDING FIX THIS
             repository = repository.cast()
             try:
                 blob = repository.pending_blobs.get(digest=pk)
                 blob.touch()
             except models.Blob.DoesNotExist:
-                raise BlobNotFound(digest=pk)
-                #try:
-                #    blob = models.ConfigBlob.objects.get(digest=pk,pk__in=repository_version.content)
-                #except:
-                #    raise BlobNotFound(digest=pk)
-                #    # pass
+                try:
+                    blob = repository.pending_config_blobs.get(digest=pk)
+                    blob.touch()
+                except:
+                    raise BlobNotFound(digest=pk)
+                    # pass
 
 #        try:
 #            config_blob = models.ConfigBlob.get(digest=pk,pk__in=repository_version.content)
@@ -1176,7 +1212,8 @@ def put(self, request, path, pk=None):
         latest_version_content_pks = repository.latest_version().content.values_list("pk")
         manifests_pks = repository.pending_manifests.values_list("pk")
         blobs_pks = repository.pending_blobs.values_list("pk")
-        content_pks = latest_version_content_pks.union(manifests_pks).union(blobs_pks)
+        config_blobs_pks = repository.pending_config_blobs.values_list("pk")
+        content_pks = latest_version_content_pks.union(manifests_pks).union(blobs_pks).union(config_blobs_pks)
 
         found_manifests = models.Manifest.objects.none()
 
@@ -1224,7 +1261,7 @@ def put(self, request, path, pk=None):
                 digest__in=found_manifests.values_list("blobs__digest"),
                 pk__in=content_pks,
             )
-            found_config_blobs = models.Blob.objects.filter(
+            found_config_blobs = models.ConfigBlob.objects.filter(
                 digest__in=found_manifests.values_list("config_blob__digest"),
                 pk__in=content_pks,
             )
@@ -1238,8 +1275,8 @@ def put(self, request, path, pk=None):
                 )
 
             config_digest = config_layer.get("digest")
-            found_config_blobs = models.Blob.objects.filter(
-                digest=config_digest, pk__in=content_pks
+            found_config_blobs = models.ConfigBlob.objects.filter(
+                digest=config_digest
             )
             if not found_config_blobs.exists():
                 raise BlobInvalid(digest=config_digest)

pulp_container/tests/functional/api/test_sync.py

@@ -61,6 +61,10 @@ def test_sync_labelled_image(
     container_tag_api,
     container_manifest_api,
 ):
+
+    #import debugpy
+    #debugpy.listen(('0.0.0.0',5678))
+    #debugpy.wait_for_client()
     """Test syncing an image containing labels and assert on their availability in the ViewSet."""
     remote = container_remote_factory(upstream_name=PULP_LABELED_FIXTURE)
     repo_version = container_sync(container_repo, remote).created_resources[0]