From f377a9cfa80ca74c70b03e585456ce40f80d2b03 Mon Sep 17 00:00:00 2001 From: Ina Panova Date: Tue, 26 Mar 2024 17:15:35 +0100 Subject: [PATCH] Various fixes to the handle-image-data command closes #1573 closes #1575 * taught command to skip content that has annotations/labels populated * got rid of Paginator. Paginator is lazy, when items are updated, it will make an new query for the page, and skip objects. --- CHANGES/1573.bugfix | 1 + CHANGES/1575.bugfix | 1 + .../commands/container-handle-image-data.py | 46 +++++++++---------- 3 files changed, 25 insertions(+), 23 deletions(-) create mode 100644 CHANGES/1573.bugfix create mode 100644 CHANGES/1575.bugfix diff --git a/CHANGES/1573.bugfix b/CHANGES/1573.bugfix new file mode 100644 index 000000000..5fda354f0 --- /dev/null +++ b/CHANGES/1573.bugfix @@ -0,0 +1 @@ +Fixed hande-image-data command to skip content that has labels/annotations already populated. diff --git a/CHANGES/1575.bugfix b/CHANGES/1575.bugfix new file mode 100644 index 000000000..15e1bc985 --- /dev/null +++ b/CHANGES/1575.bugfix @@ -0,0 +1 @@ +Fixed handle-image-data command to update all entries in one run. diff --git a/pulp_container/app/management/commands/container-handle-image-data.py b/pulp_container/app/management/commands/container-handle-image-data.py index b65fc183b..ed10e7547 100644 --- a/pulp_container/app/management/commands/container-handle-image-data.py +++ b/pulp_container/app/management/commands/container-handle-image-data.py @@ -6,14 +6,11 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.management import BaseCommand -from django.core.paginator import Paginator from pulp_container.app.models import Manifest from pulp_container.constants import MEDIA_TYPE -PAGE_CHUNK_SIZE = 1000 - class Command(BaseCommand): """ @@ -34,42 +31,45 @@ class Command(BaseCommand): def handle(self, *args, **options): manifests_updated_count = 0 - manifests = Manifest.objects.exclude( + manifests = Manifest.objects.filter(labels={}, annotations={}) + manifests = manifests.exclude( media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_V1] - ).order_by("pulp_id") + ) manifests_updated_count += self.update_manifests(manifests) manifest_lists = Manifest.objects.filter( - media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI] - ).order_by("pulp_id") + media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI], annotations={} + ) manifests_updated_count += self.update_manifests(manifest_lists) self.stdout.write( - self.style.SUCCESS("Successfully handled %d manifests." % manifests_updated_count) + self.style.SUCCESS("Successfully updated %d manifests." % manifests_updated_count) ) def update_manifests(self, manifests_qs): manifests_updated_count = 0 - - paginator = Paginator(manifests_qs, PAGE_CHUNK_SIZE) - for page_num in paginator.page_range: - manifests_to_update = [] - - page = paginator.page(page_num) - for manifest in page.object_list: - # suppress non-existing/already migrated artifacts and corrupted JSON files - with suppress(ObjectDoesNotExist, JSONDecodeError): - has_metadata = manifest.init_metadata() - if has_metadata: - manifests_to_update.append(manifest) - - if manifests_to_update: + manifests_to_update = [] + for manifest in manifests_qs.iterator(): + # suppress non-existing/already migrated artifacts and corrupted JSON files + with suppress(ObjectDoesNotExist, JSONDecodeError): + has_metadata = manifest.init_metadata() + if has_metadata: + manifests_to_update.append(manifest) + + if len(manifests_to_update) > 1000: fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak"] manifests_qs.model.objects.bulk_update( manifests_to_update, fields_to_update, ) - manifests_updated_count += len(manifests_to_update) + manifests_to_update.clear() + if manifests_to_update: + fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak"] + manifests_qs.model.objects.bulk_update( + manifests_to_update, + fields_to_update, + ) + manifests_updated_count += len(manifests_to_update) return manifests_updated_count