Improve group package count api call (#611)

* [sitemap]: remove diallow all from sitemap

* empty

* improve groups package count fetch

deployment/frontend/src/interfaces/search.interface.ts

@@ -6,6 +6,10 @@ export type Facets = Record<
     }
 >
 
+export type FacetsCount = Record<
+    string, Record<string, number>
+>
+
 export interface Filter {
     title: string // E.g. Team
     key: String // E.g. organization

deployment/frontend/src/server/api/routers/teams.ts

@@ -22,6 +22,7 @@ import {
     searchHierarchy,
     findAllNameInTree,
     getAllDatasetFq,
+    fetchFacets
 } from '@/utils/apiUtils'
 import { findNameInTree, sendMemberNotifications } from '@/utils/apiUtils'
 import { json } from 'stream/consumers'
@@ -327,15 +328,12 @@ export const teamRouter = createTRPCRouter({
                 },
                 {} as Record<string, GroupsmDetails>
             )
+
+            const facets = await fetchFacets(teamDetails, "organization", ctx?.session?.user.apikey ?? '')
 
             for (const group in teamDetails) {
-                const team = teamDetails[group]!
-                const packagedetails = (await getAllDatasetFq({
-                    apiKey: ctx?.session?.user.apikey ?? '',
-                    fq: `organization:${team.name}+is_approved:true`,
-                    query: { search: '', page: { start: 0, rows: 10000 } },
-                }))!
-                team.package_count = packagedetails.count
+                const team = teamDetails[group]!;
+                team.package_count = facets[team.name] ?? 0;
             }
 
             const result = groupTree

deployment/frontend/src/server/api/routers/topics.ts

@@ -13,6 +13,7 @@ import {
     findAllNameInTree,
     getAllDatasetFq,
     groupList,
+    fetchFacets,
 } from '@/utils/apiUtils'
 import { searchSchema } from '@/schema/search.schema'
 import type {
@@ -130,15 +131,10 @@ export const TopicRouter = createTRPCRouter({
             {} as Record<string, GroupsmDetails>
         )
         if (user) {
+            const facets = await fetchFacets(topicDetails, "groups", ctx?.session?.user.apikey ?? '')
             for (const group in topicDetails) {
                 const topic = topicDetails[group]!
-                if (topic.package_count === 0) continue;
-                const packagedetails = (await getAllDatasetFq({
-                    apiKey: ctx?.session?.user.apikey ?? '',
-                    fq: `groups:${topic.name}+is_approved:true`,
-                    query: { search: '', page: { start: 0, rows: 1 } },
-                }))!
-                topic.package_count = packagedetails.count
+                topic.package_count = facets[topic.name] ?? 0;
             }
         }
         return {
@@ -397,14 +393,11 @@ export const TopicRouter = createTRPCRouter({
                 {} as Record<string, GroupsmDetails>
             )
 
+            const facets = await fetchFacets(topicDetails, "groups", ctx?.session?.user.apikey ?? '')
+
             for (const group in topicDetails) {
-                const topic = topicDetails[group]!
-                const packagedetails = (await getAllDatasetFq({
-                    apiKey: ctx?.session?.user.apikey ?? '',
-                    fq: `groups:${topic.name}+is_approved:true`,
-                    query: { search: '', page: { start: 0, rows: 1 } },
-                }))!
-                topic.package_count = packagedetails.count
+                const topic = topicDetails[group]!;
+                topic.package_count = facets[topic.name] ?? 0;
             }
 
             const result = groupTree

deployment/frontend/src/utils/apiUtils.ts

@@ -16,7 +16,7 @@ import type {
 } from '@/schema/ckan.schema'
 import type { Group } from '@portaljs/ckan'
 import type { SearchInput } from '@/schema/search.schema'
-import { Facets, Filter } from '@/interfaces/search.interface'
+import { Facets, FacetsCount, Filter } from '@/interfaces/search.interface'
 import { replaceNames } from '@/utils/replaceNames'
 import { Session } from 'next-auth'
 import nodemailer from 'nodemailer'
@@ -316,7 +316,7 @@ export async function getAllDatasetFq({
     extAddressQ?: string
     extGlobalQ?: string
     user?: boolean | null
-}): Promise<{ datasets: WriDataset[]; count: number; searchFacets: Facets }> {
+}): Promise<{ datasets: WriDataset[]; count: number; searchFacets: Facets; facets: FacetsCount }> {
     try {
         let url = `${env.CKAN_URL}/api/3/action/package_search?q=${query.search}`
 
@@ -364,6 +364,7 @@ export async function getAllDatasetFq({
             results: WriDataset[]
             count: number
             search_facets: Facets
+            facets: FacetsCount
         }>
 
         if (data.error) {
@@ -373,10 +374,11 @@ export async function getAllDatasetFq({
         const datasets = data.success === true ? data.result.results : []
 
         const count = data.success === true ? data.result.count : 0
+        const facets = data.success === true ? data.result.facets : {}
         const searchFacets =
             data.success === true ? data.result?.search_facets : {}
 
-        return { datasets, count, searchFacets }
+        return { datasets, count, searchFacets, facets }
     } catch (e) {
         console.error(e)
         throw new Error('Failed to fetch datasets')
@@ -985,15 +987,11 @@ export async function getOrganizationTreeDetails({
         },
         {} as Record<string, GroupsmDetails>
     )
-
+
+    const facets = await fetchFacets(teamDetails, 'organization', session?.user.apikey ?? '');
     for (const group in teamDetails) {
-        const team = teamDetails[group]!
-        const packagedetails = (await getAllDatasetFq({
-            apiKey: session?.user.apikey ?? '',
-            fq: `organization:${team.name}+is_approved:true`,
-            query: { search: '', page: { start: 0, rows: 10000 } },
-        }))!
-        team.package_count = packagedetails.count
+        const team = teamDetails[group]!;
+        team.package_count = facets[team.name] ?? 0;
     }
 
     const result = groupTree
@@ -1004,6 +1002,23 @@ export async function getOrganizationTreeDetails({
     }
 }
 
+export async function fetchFacets(
+    teamDetails: Record<string, { name: string }>,
+    groupType: 'organization' | 'groups',
+    apiKey: string
+): Promise<Record<string, number>> {
+    const fq = `(${Object.values(teamDetails).map(item => item.name).join(' OR ')})`;
+
+    const facetsQuery = await getAllDatasetFq({
+        apiKey: apiKey,
+        fq: `${groupType}:${fq}+is_approved:true`,
+        facetFields: [groupType],
+        query: { search: '', page: { start: 0, rows: 0 } },
+    });
+
+    return facetsQuery.facets[groupType] ?? {};
+}
+
 export async function getTopicTreeDetails({
     input,
     session,
@@ -1063,14 +1078,11 @@ export async function getTopicTreeDetails({
         {} as Record<string, GroupsmDetails>
     )
 
+    const facets = await fetchFacets(topicDetails, 'groups', session?.user.apikey ?? '');
+
     for (const group in topicDetails) {
-        const topic = topicDetails[group]!
-        const packagedetails = (await getAllDatasetFq({
-            apiKey: session?.user.apikey ?? '',
-            fq: `groups:${topic.name}+is_approved:true`,
-            query: { search: '', page: { start: 0, rows: 10000 } },
-        }))!
-        topic.package_count = packagedetails.count
+        const topic = topicDetails[group]!;
+        topic.package_count = facets[topic.name] ?? 0;
     }
 
     const result = groupTree
@@ -2280,7 +2292,9 @@ export async function approvePendingDataset(
         rw_id = datasetRw.data.id
     }
 
-    const hasLayersToEdit = submittedDataset.resources.some(l => l.rw_id && l.url)
+    const hasLayersToEdit = submittedDataset.resources.some(
+        (l) => l.rw_id && l.url
+    )
     const resourcesToEditLayer = hasLayersToEdit
         ? await Promise.all(
               submittedDataset.resources
@@ -2705,9 +2719,8 @@ export function advance_search_query(filters: Filter[]) {
                 ? metadataModifiedBeforeFilter.value + 'T23:59:59Z'
                 : '*'
 
-            fq[
-                'metadata_modified'
-            ] = `[${metadataModifiedSince} TO ${metadataModifiedBefore}]`
+            fq['metadata_modified'] =
+                `[${metadataModifiedSince} TO ${metadataModifiedBefore}]`
         } else if (key == 'spatial') {
             const coordinates = keyFilters[0]?.value
             const address = keyFilters[0]?.label