Fixes #36702 - store package counts after smart proxy sync

app/controllers/katello/api/v2/capsule_content_controller.rb

@@ -20,6 +20,19 @@ class Api::V2::CapsuleContentController < Api::V2::ApiController
       param :environment_id, Integer, :desc => N_('Id of the lifecycle environment'), :required => true
     end
 
+    api :GET, '/capsules/:id/content/counts', N_('List content counts for the smart proxy')
+    param :id, Integer, :desc => N_('Id of the smart proxy'), :required => true
+    def counts
+      render json: @capsule.content_counts.to_json
+    end
+
+    api :POST, '/capsules/:id/content/update_counts', N_('Update content counts for the smart proxy')
+    param :id, Integer, :desc => N_('Id of the smart proxy'), :required => true
+    def update_counts
+      task = async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, @capsule)
+      respond_for_async :resource => task
+    end
+
     api :GET, '/capsules/:id/content/lifecycle_environments', N_('List the lifecycle environments attached to the smart proxy')
     param_group :lifecycle_environments
     def lifecycle_environments

app/lib/actions/katello/capsule_content/sync_capsule.rb

@@ -3,6 +3,7 @@ module Katello
     module CapsuleContent
       class SyncCapsule < ::Actions::EntryAction
         # rubocop:disable Metrics/MethodLength
+        execution_plan_hooks.use :update_content_counts, :on => :success
         def plan(smart_proxy, options = {})
           plan_self(:smart_proxy_id => smart_proxy.id)
           action_subject(smart_proxy)
@@ -60,6 +61,11 @@ def repos_to_sync(smart_proxy, environment, content_view, repository, skip_metat
           end
         end
 
+        def update_content_counts(_execution_plan)
+          smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
+          ::ForemanTasks.async_task(::Actions::Katello::CapsuleContent::UpdateContentCounts, smart_proxy)
+        end
+
         def resource_locks
           :link
         end

app/lib/actions/katello/capsule_content/update_content_counts.rb

@@ -0,0 +1,20 @@
+module Actions
+  module Katello
+    module CapsuleContent
+      class UpdateContentCounts < Actions::EntryAction
+        def plan(smart_proxy)
+          plan_self(:smart_proxy_id => smart_proxy.id)
+        end
+
+        def humanized_name
+          _("Update Content Counts")
+        end
+
+        def run
+          smart_proxy = ::SmartProxy.unscoped.find(input[:smart_proxy_id])
+          smart_proxy.update_content_counts!
+        end
+      end
+    end
+  end
+end

app/models/katello/concerns/smart_proxy_extensions.rb

@@ -121,6 +121,36 @@ def alternate_content_sources
         SmartProxy.joins(:smart_proxy_alternate_content_sources).where('katello_smart_proxy_alternate_content_sources.smart_proxy_id' => self.id)
       end
 
+      def update_content_counts!
+        # {:content_view_versions=>{87=>{:repositories=>{1=>{:rpms=>98, :module_streams=>9898}}}}}
+        new_content_counts = { content_view_versions: {} }
+        smart_proxy_helper = ::Katello::SmartProxyHelper.new(self)
+        repos = smart_proxy_helper.repositories_available_to_capsule
+        return new_content_counts if repos.empty?
+
+        repos.each do |repo|
+          repo_mirror_service = repo.backend_service(self).with_mirror_adapter
+          repo_content_counts = repo_mirror_service.latest_content_counts
+          translated_counts = {}
+          repo_content_counts.each do |name, count|
+            count = count[:count]
+            # Some content units in Pulp have the same model
+            if name == 'rpm.package' && repo.content_counts['srpm'] > 0
+              translated_counts['srpm'] = repo_mirror_service.count_by_pulpcore_type(::Katello::Pulp3::Srpm)
+              translated_counts['rpm'] = count - translated_counts['srpm']
+            elsif name == 'container.manifest' && repo.content_counts['docker_manifest_list'] > 0
+              translated_counts['docker_manifest_list'] = repo_mirror_service.count_by_pulpcore_type(::Katello::Pulp3::DockerManifestList)
+              translated_counts['docker_manifest'] = count - translated_counts['docker_manifest_list']
+            else
+              translated_counts[::Katello::Pulp3::PulpContentUnit.katello_name_from_pulpcore_name(name, repo)] = count
+            end
+          end
+          new_content_counts[:content_view_versions][repo.content_view_version_id] ||= { repositories: {} }
+          new_content_counts[:content_view_versions][repo.content_view_version_id][:repositories][repo.id] = translated_counts
+        end
+        update(content_counts: new_content_counts)
+      end
+
       def sync_container_gateway
         if has_feature?(::SmartProxy::CONTAINER_GATEWAY_FEATURE)
           update_container_repo_list

app/services/katello/pulp3/ansible_collection.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class AnsibleCollection < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "ansible.collection".freeze
 
       def self.content_api
         PulpAnsibleClient::ContentCollectionVersionsApi.new(Katello::Pulp3::Api::AnsibleCollection.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/deb.rb

@@ -3,6 +3,7 @@ module Pulp3
     class Deb < PulpContentUnit
       include LazyAccessor
       CONTENT_TYPE = "deb".freeze
+      PULPCORE_CONTENT_TYPE = "deb.package".freeze
 
       def self.content_api
         PulpDebClient::ContentPackagesApi.new(Katello::Pulp3::Api::Apt.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/docker_manifest.rb

@@ -3,6 +3,7 @@ module Pulp3
     class DockerManifest < PulpContentUnit
       include LazyAccessor
       CONTENT_TYPE = "docker_manifest".freeze
+      PULPCORE_CONTENT_TYPE = "container.manifest".freeze
 
       def self.content_api
         PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/docker_manifest_list.rb

@@ -2,9 +2,10 @@ module Katello
   module Pulp3
     class DockerManifestList < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "container.manifest".freeze
 
-      def self.content_api
-        PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)
+      def self.content_api(smart_proxy = SmartProxy.pulp_primary!)
+        PulpContainerClient::ContentManifestsApi.new(Katello::Pulp3::Api::Docker.new(smart_proxy).api_client)
       end
 
       def self.ids_for_repository(repo_id)
@@ -13,10 +14,14 @@ def self.ids_for_repository(repo_id)
         repo_content_list.map { |content| content.try(:pulp_href) }
       end
 
-      def self.content_unit_list(page_opts)
+      def self.page_options(page_opts = {})
         page_opts[:media_type] = ['application/vnd.docker.distribution.manifest.list.v2+json',
                                   'application/vnd.oci.image.index.v1+json']
-        self.content_api.list(page_opts)
+        page_opts
+      end
+
+      def self.content_unit_list(page_opts = {})
+        self.content_api.list(self.page_options(page_opts))
       end
 
       def self.generate_model_row(unit)

app/services/katello/pulp3/docker_tag.rb

@@ -3,6 +3,7 @@ module Pulp3
     class DockerTag < PulpContentUnit
       include LazyAccessor
       CONTENT_TYPE = "docker_tag".freeze
+      PULPCORE_CONTENT_TYPE = "container.tag".freeze
 
       def self.content_api
         PulpContainerClient::ContentTagsApi.new(Katello::Pulp3::Api::Docker.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/erratum.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class Erratum < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "rpm.advisory".freeze
 
       def self.content_api
         PulpRpmClient::ContentAdvisoriesApi.new(Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/file_unit.rb

@@ -3,6 +3,7 @@ module Pulp3
     class FileUnit < PulpContentUnit
       include LazyAccessor
       CONTENT_TYPE = "file".freeze
+      PULPCORE_CONTENT_TYPE = "file.file".freeze
 
       def self.content_api
         PulpFileClient::ContentFilesApi.new(Katello::Pulp3::Api::File.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/module_stream.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class ModuleStream < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "rpm.modulemd".freeze
 
       def self.content_api
         PulpRpmClient::ContentModulemdsApi.new(Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_primary!).api_client)

app/services/katello/pulp3/package_group.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class PackageGroup < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "rpm.packagegroup".freeze
 
       lazy_accessor :optional_package_names, :mandatory_package_names,
                     :conditional_package_names, :default_package_names, :_id,

app/services/katello/pulp3/pulp_content_unit.rb

@@ -7,6 +7,41 @@ class PulpContentUnit
       # Any class that extends this class should define:
       # Class#update_model
 
+      def self.katello_name_from_pulpcore_name(pulpcore_name, repo)
+        # Counts shouldn't be needed for more than the default generic content unit type.
+        if repo.generic?
+          generic_unit = repo.repository_type.default_managed_content_type
+          if pulpcore_name == generic_unit.pulpcore_name
+            return generic_unit.content_type
+          end
+        end
+
+        case pulpcore_name
+        when ::Katello::Pulp3::Rpm::PULPCORE_CONTENT_TYPE
+          ::Katello::Rpm::CONTENT_TYPE
+        when ::Katello::Pulp3::Srpm::PULPCORE_CONTENT_TYPE
+          ::Katello::Srpm::CONTENT_TYPE
+        when ::Katello::Pulp3::PackageGroup::PULPCORE_CONTENT_TYPE
+          ::Katello::PackageGroup::CONTENT_TYPE
+        when ::Katello::Pulp3::Erratum::PULPCORE_CONTENT_TYPE
+          ::Katello::Erratum::CONTENT_TYPE
+        when ::Katello::Pulp3::DockerTag::PULPCORE_CONTENT_TYPE
+          ::Katello::DockerTag::CONTENT_TYPE
+        when ::Katello::Pulp3::DockerManifest::PULPCORE_CONTENT_TYPE
+          ::Katello::DockerManifest::CONTENT_TYPE
+        when ::Katello::Pulp3::DockerManifestList::PULPCORE_CONTENT_TYPE
+          ::Katello::DockerManifestList::CONTENT_TYPE
+        when ::Katello::Pulp3::FileUnit::PULPCORE_CONTENT_TYPE
+          ::Katello::FileUnit::CONTENT_TYPE
+        when ::Katello::Pulp3::Deb::PULPCORE_CONTENT_TYPE
+          ::Katello::Deb::CONTENT_TYPE
+        when ::Katello::Pulp3::AnsibleCollection::PULPCORE_CONTENT_TYPE
+          ::Katello::AnsibleCollection::CONTENT_TYPE
+        else
+          pulpcore_name
+        end
+      end
+
       def self.content_api
         fail NotImplementedError
       end

app/services/katello/pulp3/repository_mirror.rb

@@ -211,6 +211,18 @@ def refresh_distributions(_options = {})
         end
       end
 
+      def count_by_pulpcore_type(service_class)
+        # Currently only supports SRPMs and Docker Manifest Lists
+        fail NotImplementedError unless [::Katello::Pulp3::Srpm, ::Katello::Pulp3::DockerManifestList].include?(service_class)
+        count = service_class.content_api(smart_proxy).list(service_class.page_options(limit: 1, fields: ['count'], repository_version: version_href)).count
+        return 0 if count.nil?
+        count
+      end
+
+      def latest_content_counts
+        api.repository_versions_api.read(version_href)&.content_summary&.present
+      end
+
       def pulp3_enabled_repo_types
         Katello::RepositoryTypeManager.enabled_repository_types.values.select do |repository_type|
           smart_proxy.pulp3_repository_type_support?(repository_type)

app/services/katello/pulp3/rpm.rb

@@ -3,6 +3,7 @@ module Pulp3
     class Rpm < PulpContentUnit
       include LazyAccessor
       CONTENT_TYPE = "rpm".freeze
+      PULPCORE_CONTENT_TYPE = "rpm.package".freeze
 
       PULP_INDEXED_FIELDS = %w(pulp_href name version release arch epoch summary is_modular rpm_sourcerpm location_href pkgId).freeze
 

app/services/katello/pulp3/srpm.rb

@@ -2,6 +2,7 @@ module Katello
   module Pulp3
     class Srpm < PulpContentUnit
       include LazyAccessor
+      PULPCORE_CONTENT_TYPE = "rpm.package".freeze
 
       CONTENT_TYPE = "srpm".freeze
 
@@ -13,8 +14,8 @@ class Srpm < PulpContentUnit
                     :changelog, :group, :size, :url, :build_time, :group,
                     :initializer => :pulp_facts
 
-      def self.content_api
-        PulpRpmClient::ContentPackagesApi.new(Katello::Pulp3::Api::Yum.new(SmartProxy.pulp_primary!).api_client)
+      def self.content_api(smart_proxy = SmartProxy.pulp_primary!)
+        PulpRpmClient::ContentPackagesApi.new(Katello::Pulp3::Api::Yum.new(smart_proxy).api_client)
       end
 
       def self.page_options(page_opts = {})

app/services/katello/repository_type.rb

@@ -181,7 +181,7 @@ def generic?
 
     class GenericContentType < ContentType
       attr_accessor :pulp3_api, :pulp3_model, :filename_key, :duplicates_allowed, :pluralized_name,
-                    :model_name, :model_version, :model_filename, :model_additional_metadata
+                    :model_name, :model_version, :model_filename, :model_additional_metadata, :pulpcore_name
 
       def initialize(options)
         super
@@ -195,6 +195,7 @@ def initialize(options)
         self.model_version = options[:model_version]
         self.model_filename = options[:model_filename]
         self.model_additional_metadata = options[:model_additional_metadata]
+        self.pulpcore_name = options[:pulpcore_name]
       end
 
       def label

config/routes/api/v2.rb

@@ -23,6 +23,8 @@ class ActionDispatch::Routing::Mapper
             resource :content, :only => [], :controller => 'capsule_content' do
               get :lifecycle_environments
               get :available_lifecycle_environments
+              get :counts
+              post :update_counts
               post :sync
               get :sync, :action => :sync_status
               delete :sync, :action => :cancel_sync

db/migrate/20230825180856_add_content_counts_to_smart_proxy.rb

@@ -0,0 +1,7 @@
+class AddContentCountsToSmartProxy < ActiveRecord::Migration[6.1]
+  def change
+    # {:content_view_versions=>{87=>{:repositories=>{1=>{:rpms=>98, :module_streams=>9898}, 2=>{:tags=>32432, :manifests=>323}}}}}
+    add_column :smart_proxies, :content_counts, :jsonb
+    add_index :smart_proxies, :content_counts
+  end
+end

lib/katello/permission_creator.rb

@@ -54,14 +54,14 @@ def capsule_content_permissions
       @plugin.permission :manage_capsule_content,
                          {
                            'katello/api/v2/capsule_content' => [:add_lifecycle_environment, :remove_lifecycle_environment,
-                                                                :sync, :reclaim_space, :cancel_sync],
+                                                                :update_counts, :sync, :reclaim_space, :cancel_sync],
                            'katello/api/v2/capsules' => [:index, :show]
                          },
                          :resource_type => 'SmartProxy'
 
       @plugin.permission :view_capsule_content,
                          {
-                           'katello/api/v2/capsule_content' => [:lifecycle_environments, :available_lifecycle_environments, :sync_status],
+                           'katello/api/v2/capsule_content' => [:counts, :lifecycle_environments, :available_lifecycle_environments, :sync_status],
                            'smart_proxies' => [:pulp_storage, :pulp_status, :show_with_content]
                          },
                          :resource_type => "SmartProxy"

lib/katello/repository_types/ostree.rb

@@ -22,6 +22,7 @@
 
   generic_content_type 'ostree_ref',
                        pluralized_name: "OSTree Refs",
+                       pulpcore_name: "ostree.refs",
                        model_class: Katello::GenericContentUnit,
                        pulp3_api: PulpOstreeClient::ContentRefsApi,
                        pulp3_service_class: Katello::Pulp3::GenericContentUnit,

lib/katello/repository_types/python.rb

@@ -32,6 +32,7 @@
 
   generic_content_type 'python_package',
                        pluralized_name: "Python Packages",
+                       pulpcore_name: "python.python",
                        model_class: Katello::GenericContentUnit,
                        pulp3_api: PulpPythonClient::ContentPackagesApi,
                        pulp3_model: PulpPythonClient::PythonPythonPackageContent,

test/controllers/api/v2/capsule_content_controller_test.rb

@@ -133,6 +133,21 @@ def test_cancel_sync_protected
       end
     end
 
+    def test_update_counts
+      assert_async_task ::Actions::Katello::CapsuleContent::UpdateContentCounts do |capsule|
+        assert_equal proxy_with_pulp.id, capsule.id
+      end
+
+      post :update_counts, params: { :id => proxy_with_pulp.id }
+      assert_response :success
+    end
+
+    def test_counts
+      SmartProxy.any_instance.expects(:content_counts).once
+      get :counts, params: { :id => proxy_with_pulp.id }
+      assert_response :success
+    end
+
     def test_reclaim_space
       assert_async_task ::Actions::Pulp3::CapsuleContent::ReclaimSpace do |capsule|
         assert_equal proxy_with_pulp.id, capsule.id