From 0978d7df98b697b78f696bff08ea7dade2c124e6 Mon Sep 17 00:00:00 2001 From: git-hyagi <45576767+git-hyagi@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:30:03 -0300 Subject: [PATCH] [draft/wip] image push --- ...nd_more.py => 0039_configblob_and_more.py} | 13 +++- pulp_container/app/models.py | 2 + pulp_container/app/registry.py | 3 +- pulp_container/app/registry_api.py | 69 ++++++++++++++----- 4 files changed, 69 insertions(+), 18 deletions(-) rename pulp_container/app/migrations/{0039_configblob_alter_manifest_config_blob_and_more.py => 0039_configblob_and_more.py} (82%) diff --git a/pulp_container/app/migrations/0039_configblob_alter_manifest_config_blob_and_more.py b/pulp_container/app/migrations/0039_configblob_and_more.py similarity index 82% rename from pulp_container/app/migrations/0039_configblob_alter_manifest_config_blob_and_more.py rename to pulp_container/app/migrations/0039_configblob_and_more.py index add20a524..d03ea284d 100644 --- a/pulp_container/app/migrations/0039_configblob_alter_manifest_config_blob_and_more.py +++ b/pulp_container/app/migrations/0039_configblob_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-03-19 15:16 +# Generated by Django 4.2.10 on 2024-03-21 19:41 from django.db import migrations, models import django.db.models.deletion @@ -7,6 +7,7 @@ class Migration(migrations.Migration): dependencies = [ + ('core', '0117_task_unblocked_at'), ('container', '0038_add_manifest_metadata_fields'), ] @@ -34,6 +35,16 @@ class Migration(migrations.Migration): }, bases=('core.content',), ), + migrations.AddField( + model_name='containerpushrepository', + name='pending_config_blobs', + field=models.ManyToManyField(to='container.configblob'), + ), + migrations.AddField( + model_name='containerrepository', + name='pending_config_blobs', + field=models.ManyToManyField(to='container.configblob'), + ), migrations.AlterField( model_name='manifest', name='config_blob', diff --git a/pulp_container/app/models.py b/pulp_container/app/models.py index c9dcfce08..362bb87d4 100644 --- a/pulp_container/app/models.py +++ b/pulp_container/app/models.py @@ -645,6 +645,7 @@ class ContainerRepository( ManifestSigningService, on_delete=models.SET_NULL, null=True ) pending_blobs = models.ManyToManyField(Blob) + pending_config_blobs = models.ManyToManyField(ConfigBlob) pending_manifests = models.ManyToManyField(Manifest) class Meta: @@ -702,6 +703,7 @@ class ContainerPushRepository(Repository, AutoAddObjPermsMixin): ManifestSigningService, on_delete=models.SET_NULL, null=True ) pending_blobs = models.ManyToManyField(Blob) + pending_config_blobs = models.ManyToManyField(ConfigBlob) pending_manifests = models.ManyToManyField(Manifest) class Meta: diff --git a/pulp_container/app/registry.py b/pulp_container/app/registry.py index e8ec01d14..96124a5ed 100644 --- a/pulp_container/app/registry.py +++ b/pulp_container/app/registry.py @@ -424,8 +424,9 @@ async def run_pipeline(self, saved_artifact): async def init_pending_content(self, digest, manifest_data, media_type, artifact): if config := manifest_data.get("config", None): config_digest = config["digest"] + config_blob = await self.save_config_blob(config_digest) - await sync_to_async(self.repository.pending_blobs.add)(config_blob) + await sync_to_async(self.repository.pending_config_blobs.add)(config_blob) else: config_blob = None diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 9f7506265..30a7c4c80 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -771,10 +771,6 @@ def create_single_chunk_artifact(self, chunk): artifact.touch() return artifact - ### CHECK IF THIS WILL NEED TO BE MODIFIED - ### CHECK IF THIS WILL NEED TO BE MODIFIED - ### CHECK IF THIS WILL NEED TO BE MODIFIED - ### CHECK IF THIS WILL NEED TO BE MODIFIED def create_blob(self, artifact, digest): with transaction.atomic(): try: @@ -872,6 +868,38 @@ def partial_update(self, request, path, pk=None): return UploadResponse(upload=upload, path=path, request=request, status=204) + + def is_config_blob(self,blob): + required_fields = { "os", "rootfs", "architecture"} + if all(config_key in required_fields for config_key in blob): + return True + return False + + + def create_config_blob(self, manifest,raw_manifest, digest): + with transaction.atomic(): + try: + blob = models.ConfigBlob( + data = raw_manifest, + digest = digest, + architecture=manifest.get("architecture"), + os=manifest.get("os"), + created=manifest.get("created"), + author=manifest.get("author", ""), + os_version=manifest.get("os_version", ""), + os_features=manifest.get("os_features", ""), + variant=manifest.get("variant", ""), + rootfs=manifest.get("rootfs", {}), + config=manifest.get("config", {}), + history=manifest.get("history", {}), + ) + blob.save() + except IntegrityError: + blob = models.ConfigBlob.objects.get(digest=digest) + blob.touch() + return blob + + def put(self, request, path, pk=None): """ Create a blob from uploaded chunks. @@ -917,10 +945,18 @@ def put(self, request, path, pk=None): artifact = Artifact.objects.get(sha256=artifact.sha256) artifact.touch() - blob = self.create_blob(artifact, digest) + blob = None + try: + raw_manifest = artifact.file.read().decode("utf-8") + manifest = json.loads(raw_manifest) + blob = self.create_config_blob(manifest, raw_manifest, digest) + repository.pending_config_blobs.add(blob) + except UnicodeDecodeError: + blob = self.create_blob(artifact, digest) + repository.pending_blobs.add(blob) upload.delete() - repository.pending_blobs.add(blob) + return BlobResponse(blob, path, 201, request) @@ -983,18 +1019,18 @@ def handle_safe_method(self, request, path, pk): blob = models.ConfigBlob.objects.get(digest=pk) return redirects.redirect_to_content_app("config-blobs",pk) except: - pass + pass ###### WIP PENDING FIX THIS repository = repository.cast() try: blob = repository.pending_blobs.get(digest=pk) blob.touch() except models.Blob.DoesNotExist: - raise BlobNotFound(digest=pk) - #try: - # blob = models.ConfigBlob.objects.get(digest=pk,pk__in=repository_version.content) - #except: - # raise BlobNotFound(digest=pk) - # # pass + try: + blob = repository.pending_config_blobs.get(digest=pk) + blob.touch() + except: + raise BlobNotFound(digest=pk) + # pass # try: # config_blob = models.ConfigBlob.get(digest=pk,pk__in=repository_version.content) @@ -1167,7 +1203,8 @@ def put(self, request, path, pk=None): latest_version_content_pks = repository.latest_version().content.values_list("pk") manifests_pks = repository.pending_manifests.values_list("pk") blobs_pks = repository.pending_blobs.values_list("pk") - content_pks = latest_version_content_pks.union(manifests_pks).union(blobs_pks) + config_blobs_pks = repository.pending_config_blobs.values_list("pk") + content_pks = latest_version_content_pks.union(manifests_pks).union(blobs_pks).union(config_blobs_pks) found_manifests = models.Manifest.objects.none() @@ -1215,7 +1252,7 @@ def put(self, request, path, pk=None): digest__in=found_manifests.values_list("blobs__digest"), pk__in=content_pks, ) - found_config_blobs = models.Blob.objects.filter( + found_config_blobs = models.ConfigBlob.objects.filter( digest__in=found_manifests.values_list("config_blob__digest"), pk__in=content_pks, ) @@ -1229,7 +1266,7 @@ def put(self, request, path, pk=None): ) config_digest = config_layer.get("digest") - found_config_blobs = models.Blob.objects.filter( + found_config_blobs = models.ConfigBlob.objects.filter( digest=config_digest, pk__in=content_pks ) if not found_config_blobs.exists():