speed up pages

backend/onyx/server/documents/connector.py

@@ -17,7 +17,6 @@
 from onyx.auth.users import current_chat_accesssible_user
 from onyx.auth.users import current_curator_or_admin_user
 from onyx.auth.users import current_user
-from onyx.background.celery.celery_utils import get_deletion_attempt_snapshot
 from onyx.background.celery.versioned_apps.primary import app as primary_app
 from onyx.configs.app_configs import ENABLED_CONNECTOR_TYPES
 from onyx.configs.constants import DocumentSource
@@ -97,6 +96,7 @@
 from onyx.server.documents.models import ConnectorCredentialPairIdentifier
 from onyx.server.documents.models import ConnectorIndexingStatus
 from onyx.server.documents.models import ConnectorSnapshot
+from onyx.server.documents.models import ConnectorStatus
 from onyx.server.documents.models import ConnectorUpdateRequest
 from onyx.server.documents.models import CredentialBase
 from onyx.server.documents.models import CredentialSnapshot
@@ -497,6 +497,44 @@ def get_currently_failed_indexing_status(
     return indexing_statuses
 
 
+@router.get("/admin/connector/status")
+def get_connector_status(
+    user: User = Depends(current_curator_or_admin_user),
+    db_session: Session = Depends(get_session),
+    get_editable: bool = Query(
+        False, description="If true, return editable document sets"
+    ),
+) -> list[ConnectorStatus]:
+    cc_pairs = get_connector_credential_pairs(
+        db_session=db_session,
+        user=user,
+        get_editable=get_editable,
+    )
+
+    group_cc_pair_relationships = get_cc_pair_groups_for_ids(
+        db_session=db_session,
+        cc_pair_ids=[cc_pair.id for cc_pair in cc_pairs],
+    )
+    group_cc_pair_relationships_dict: dict[int, list[int]] = {}
+    for relationship in group_cc_pair_relationships:
+        group_cc_pair_relationships_dict.setdefault(relationship.cc_pair_id, []).append(
+            relationship.user_group_id
+        )
+
+    return [
+        ConnectorStatus(
+            cc_pair_id=cc_pair.id,
+            name=cc_pair.name,
+            connector=ConnectorSnapshot.from_connector_db_model(cc_pair.connector),
+            credential=CredentialSnapshot.from_credential_db_model(cc_pair.credential),
+            access_type=cc_pair.access_type,
+            groups=group_cc_pair_relationships_dict.get(cc_pair.id, []),
+        )
+        for cc_pair in cc_pairs
+        if cc_pair.name != "DefaultCCPair" and cc_pair.connector and cc_pair.credential
+    ]
+
+
 @router.get("/admin/connector/indexing-status")
 def get_connector_indexing_status(
     secondary_index: bool = False,
@@ -599,6 +637,7 @@ def get_connector_indexing_status(
             ConnectorIndexingStatus(
                 cc_pair_id=cc_pair.id,
                 name=cc_pair.name,
+                in_progress=in_progress,
                 cc_pair_status=cc_pair.status,
                 connector=ConnectorSnapshot.from_connector_db_model(connector),
                 credential=CredentialSnapshot.from_credential_db_model(credential),
@@ -615,30 +654,13 @@ def get_connector_indexing_status(
                 docs_indexed=cc_pair_to_document_cnt.get(
                     (connector.id, credential.id), 0
                 ),
-                error_msg=(
-                    latest_index_attempt.error_msg if latest_index_attempt else None
-                ),
                 latest_index_attempt=(
                     IndexAttemptSnapshot.from_index_attempt_db_model(
                         latest_index_attempt
                     )
                     if latest_index_attempt
                     else None
                 ),
-                deletion_attempt=get_deletion_attempt_snapshot(
-                    connector_id=connector.id,
-                    credential_id=credential.id,
-                    db_session=db_session,
-                    tenant_id=tenant_id,
-                ),
-                is_deletable=check_deletion_attempt_is_allowed(
-                    connector_credential_pair=cc_pair,
-                    db_session=db_session,
-                    # allow scheduled indexing attempts here, since on deletion request we will cancel them
-                    allow_scheduled=True,
-                )
-                is None,
-                in_progress=in_progress,
             )
         )
 

backend/onyx/server/documents/models.py

@@ -307,29 +307,31 @@ class FailedConnectorIndexingStatus(BaseModel):
     credential_id: int
 
 
-class ConnectorIndexingStatus(BaseModel):
-    """Represents the latest indexing status of a connector"""
+class ConnectorStatus(BaseModel):
+    """Represents the minimal indexing status of a connector"""
 
     cc_pair_id: int
     name: str | None
-    cc_pair_status: ConnectorCredentialPairStatus
     connector: ConnectorSnapshot
     credential: CredentialSnapshot
-    owner: str
-    groups: list[int]
     access_type: AccessType
+    groups: list[int]
+
+
+class ConnectorIndexingStatus(ConnectorStatus):
+    """Represents the full indexing status of a connector"""
+
+    cc_pair_status: ConnectorCredentialPairStatus
+    owner: str
     last_finished_status: IndexingStatus | None
     last_status: IndexingStatus | None
     last_success: datetime | None
-    docs_indexed: int
-    error_msg: str | None
     latest_index_attempt: IndexAttemptSnapshot | None
-    deletion_attempt: DeletionAttemptSnapshot | None
-    is_deletable: bool
+    docs_indexed: int
+    in_progress: bool
 
     # index attempt in db can be marked successful while celery/redis
-    # is stil running/cleaning up
-    in_progress: bool
+    # is still running/cleaning up
 
 
 class ConnectorCredentialPairIdentifier(BaseModel):

web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx

@@ -4,19 +4,15 @@ import useSWR from "swr";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { LoadingAnimation } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus } from "@/lib/types";
+import { CCPairBasicInfo } from "@/lib/types";
 import {
   Credential,
   GmailCredentialJson,
   GmailServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
 import { GmailAuthSection, GmailJsonUploadSection } from "./Credential";
-import {
-  usePublicCredentials,
-  useConnectorCredentialIndexingStatus,
-} from "@/lib/hooks";
+import { usePublicCredentials, useBasicConnectorStatus } from "@/lib/hooks";
 import Title from "@/components/ui/title";
-import { GmailConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 
 export const GmailMain = () => {
@@ -42,7 +38,7 @@ export const GmailMain = () => {
     data: connectorIndexingStatuses,
     isLoading: isConnectorIndexingStatusesLoading,
     error: connectorIndexingStatusesError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useBasicConnectorStatus();
 
   const {
     data: credentialsData,
@@ -116,13 +112,11 @@ export const GmailMain = () => {
       credential.credential_json?.google_service_account_key &&
       credential.source === "gmail"
   );
-  const gmailConnectorIndexingStatuses: ConnectorIndexingStatus<
-    GmailConfig,
-    GmailCredentialJson
-  >[] = connectorIndexingStatuses.filter(
-    (connectorIndexingStatus) =>
-      connectorIndexingStatus.connector.source === "gmail"
-  );
+
+  const gmailConnectorIndexingStatuses: CCPairBasicInfo[] =
+    connectorIndexingStatuses.filter(
+      (connectorIndexingStatus) => connectorIndexingStatus.source === "gmail"
+    );
 
   return (
     <>

web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx

@@ -8,12 +8,7 @@ import {
   updateDocumentSet,
   DocumentSetCreationRequest,
 } from "./lib";
-import {
-  ConnectorIndexingStatus,
-  DocumentSet,
-  UserGroup,
-  UserRole,
-} from "@/lib/types";
+import { ConnectorStatus, DocumentSet, UserGroup, UserRole } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { Separator } from "@/components/ui/separator";
@@ -24,7 +19,7 @@ import React, { useEffect, useState } from "react";
 import { useUser } from "@/components/user/UserProvider";
 
 interface SetCreationPopupProps {
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroups: UserGroup[] | undefined;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;

web/src/app/admin/documents/sets/[documentSetId]/page.tsx

@@ -3,10 +3,7 @@ import { use } from "react";
 
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { refreshDocumentSets, useDocumentSets } from "../hooks";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups } from "@/lib/hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BookmarkIcon } from "@/components/icons/icons";
@@ -30,7 +27,7 @@ function Main({ documentSetId }: { documentSetId: number }) {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   // EE only
   const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups();

web/src/app/admin/documents/sets/new/page.tsx

@@ -3,10 +3,7 @@
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BookmarkIcon } from "@/components/icons/icons";
 import { DocumentSetCreationForm } from "../DocumentSetCreationForm";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups } from "@/lib/hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { BackButton } from "@/components/BackButton";
@@ -23,7 +20,7 @@ function Main() {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   // EE only
   const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups();

web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx

@@ -411,11 +411,6 @@ export function CCPairIndexingStatusTable({
             last_success: "2023-07-01T12:00:00Z",
             last_finished_status: "success",
             latest_index_attempt: null,
-            owner: "1",
-            error_msg: "",
-            deletion_attempt: null,
-            is_deletable: true,
-            in_progress: false,
             groups: [], // Add this line
           }}
           isEditable={false}

web/src/app/admin/indexing/status/page.tsx

@@ -16,6 +16,7 @@ function Main() {
     isLoading: indexAttemptIsLoading,
     error: indexAttemptError,
   } = useConnectorCredentialIndexingStatus();
+
   const {
     data: editableIndexAttemptData,
     isLoading: editableIndexAttemptIsLoading,

web/src/app/ee/admin/groups/ConnectorEditor.tsx

@@ -1,10 +1,10 @@
-import { ConnectorIndexingStatus } from "@/lib/types";
+import { ConnectorIndexingStatus, ConnectorStatus } from "@/lib/types";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 
 interface ConnectorEditorProps {
   selectedCCPairIds: number[];
   setSetCCPairIds: (ccPairId: number[]) => void;
-  allCCPairs: ConnectorIndexingStatus<any, any>[];
+  allCCPairs: ConnectorStatus<any, any>[];
 }
 
 export const ConnectorEditor = ({

web/src/app/ee/admin/groups/UserGroupCreationForm.tsx

@@ -1,7 +1,7 @@
 import { Form, Formik } from "formik";
 import * as Yup from "yup";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus, User, UserGroup } from "@/lib/types";
+import { ConnectorStatus, User, UserGroup } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { createUserGroup } from "./lib";
 import { UserEditor } from "./UserEditor";
@@ -14,7 +14,7 @@ interface UserGroupCreationFormProps {
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec | null) => void;
   users: User[];
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   existingUserGroup?: UserGroup;
 }
 

web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx

@@ -5,11 +5,11 @@ import { useState } from "react";
 import { FiPlus, FiX } from "react-icons/fi";
 import { updateUserGroup } from "./lib";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { ConnectorIndexingStatus, UserGroup } from "@/lib/types";
+import { ConnectorStatus, UserGroup } from "@/lib/types";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { Connector } from "@/lib/connectors/connectors";
 interface AddConnectorFormProps {
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroup: UserGroup;
   onClose: () => void;
   setPopup: (popupSpec: PopupSpec) => void;

web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx

@@ -12,6 +12,7 @@ import {
   UserGroup,
   UserRole,
   USER_ROLE_LABELS,
+  ConnectorStatus,
 } from "@/lib/types";
 import { AddConnectorForm } from "./AddConnectorForm";
 import { Separator } from "@/components/ui/separator";
@@ -42,7 +43,7 @@ import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
 
 interface GroupDisplayProps {
   users: User[];
-  ccPairs: ConnectorIndexingStatus<any, any>[];
+  ccPairs: ConnectorStatus<any, any>[];
   userGroup: UserGroup;
   refreshUserGroup: () => void;
 }

web/src/app/ee/admin/groups/[groupId]/page.tsx

@@ -5,7 +5,11 @@ import { GroupsIcon } from "@/components/icons/icons";
 import { GroupDisplay } from "./GroupDisplay";
 import { useSpecificUserGroup } from "./hook";
 import { ThreeDotsLoader } from "@/components/Loading";
-import { useConnectorCredentialIndexingStatus, useUsers } from "@/lib/hooks";
+import {
+  useConnectorCredentialIndexingStatus,
+  useConnectorStatus,
+  useUsers,
+} from "@/lib/hooks";
 import { useRouter } from "next/navigation";
 import { BackButton } from "@/components/BackButton";
 import { AdminPageTitle } from "@/components/admin/Title";
@@ -29,7 +33,7 @@ const Page = (props: { params: Promise<{ groupId: string }> }) => {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   if (userGroupIsLoading || userIsLoading || isCCPairsLoading) {
     return (

web/src/app/ee/admin/groups/page.tsx

@@ -6,11 +6,7 @@ import { UserGroupCreationForm } from "./UserGroupCreationForm";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useState } from "react";
 import { ThreeDotsLoader } from "@/components/Loading";
-import {
-  useConnectorCredentialIndexingStatus,
-  useUserGroups,
-  useUsers,
-} from "@/lib/hooks";
+import { useConnectorStatus, useUserGroups, useUsers } from "@/lib/hooks";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { Button } from "@/components/ui/button";
 
@@ -26,7 +22,7 @@ const Main = () => {
     data: ccPairs,
     isLoading: isCCPairsLoading,
     error: ccPairsError,
-  } = useConnectorCredentialIndexingStatus();
+  } = useConnectorStatus();
 
   const {
     data: users,