[Search] Change endpoint to filter out already attached indices (elas…

…tic#178647)

## Summary



https://github.com/elastic/kibana/assets/1410658/9ab9af95-09eb-4a82-980c-f6df1daa5105

Attach box fetch is moved to a simplified copy of the same endpoint for
filtering out already attached indices. It helped also simplifying the
FE code.


### Checklist

Delete any items that are not applicable to this PR.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [x] Any UI touched in this PR is usable by keyboard only (learn more
about [keyboard accessibility](https://webaim.org/techniques/keyboard/))
- [ ] Any UI touched in this PR does not create any new axe failures
(run axe in browser:
[FF](https://addons.mozilla.org/en-US/firefox/addon/axe-devtools/),
[Chrome](https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))
- [x] This renders correctly on smaller devices using a responsive
layout. (You can test this [in your
browser](https://www.browserstack.com/guide/responsive-testing-on-local-server))
- [ ] This was checked for [cross-browser
compatibility](https://www.elastic.co/support/matrix#matrix_browsers)

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_available_indices_api_logic.ts

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Meta } from '../../../../../common/types/pagination';
+
+import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic';
+import { INPUT_THROTTLE_DELAY_MS } from '../../../shared/constants/timers';
+import { HttpLogic } from '../../../shared/http';
+
+export interface FetchAvailabeIndicesApiParams {
+  searchQuery?: string;
+}
+export interface FetchAvailableIndicesApiResponse {
+  indexNames: string[];
+  meta: Meta;
+}
+
+export const fetchAvailableIndices = async ({
+  searchQuery,
+}: FetchAvailabeIndicesApiParams): Promise<FetchAvailableIndicesApiResponse> => {
+  const { http } = HttpLogic.values;
+  const route = '/internal/enterprise_search/connectors/available_indices';
+  const query = { search_query: searchQuery || null };
+  const response = await http.get<FetchAvailableIndicesApiResponse>(route, { query });
+  return response;
+};
+
+export const FetchAvailableIndicesAPILogic = createApiLogic(
+  ['content', 'fetch_available_indices_api_logic'],
+  fetchAvailableIndices,
+  {
+    requestBreakpointMS: INPUT_THROTTLE_DELAY_MS,
+  }
+);
+
+export type FetchAvailableIndicesApiActions = Actions<
+  FetchAvailabeIndicesApiParams,
+  FetchAvailableIndicesApiResponse
+>;

x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/attach_index_box.tsx

@@ -31,7 +31,7 @@ import { Connector } from '@kbn/search-connectors';
 
 import { Status } from '../../../../../common/types/api';
 
-import { FetchAllIndicesAPILogic } from '../../api/index/fetch_all_indices_api_logic';
+import { FetchAvailableIndicesAPILogic } from '../../api/index/fetch_available_indices_api_logic';
 
 import { AttachIndexLogic } from './attach_index_logic';
 
@@ -74,13 +74,12 @@ export const AttachIndexBox: React.FC<AttachIndexBoxProps> = ({ connector }) =>
   );
   const [selectedLanguage] = useState<string>();
   const [query, setQuery] = useState<{
-    hasMatchingOptions: boolean;
     isFullMatch: boolean;
     searchValue: string;
   }>();
 
-  const { makeRequest } = useActions(FetchAllIndicesAPILogic);
-  const { data, status } = useValues(FetchAllIndicesAPILogic);
+  const { makeRequest } = useActions(FetchAvailableIndicesAPILogic);
+  const { data, status } = useValues(FetchAvailableIndicesAPILogic);
   const isLoading = [Status.IDLE, Status.LOADING].includes(status);
 
   const onSave = () => {
@@ -93,14 +92,22 @@ export const AttachIndexBox: React.FC<AttachIndexBoxProps> = ({ connector }) =>
 
   const options: Array<EuiComboBoxOptionOption<string>> = isLoading
     ? []
-    : data?.indices.map((index) => {
+    : data?.indexNames.map((name) => {
         return {
-          label: index.name,
+          label: name,
         };
       }) ?? [];
 
-  const shouldPrependUserInputAsOption =
-    !!query?.searchValue && query.hasMatchingOptions && !query.isFullMatch;
+  const hasMatchingOptions =
+    data?.indexNames.some((name) =>
+      name.toLocaleLowerCase().includes(query?.searchValue.toLocaleLowerCase() ?? '')
+    ) ?? false;
+  const isFullMatch =
+    data?.indexNames.some(
+      (name) => name.toLocaleLowerCase() === query?.searchValue.toLocaleLowerCase()
+    ) ?? false;
+
+  const shouldPrependUserInputAsOption = !!query?.searchValue && hasMatchingOptions && !isFullMatch;
 
   const groupedOptions: Array<EuiComboBoxOptionOption<string>> = shouldPrependUserInputAsOption
     ? [
@@ -127,7 +134,8 @@ export const AttachIndexBox: React.FC<AttachIndexBoxProps> = ({ connector }) =>
   }, [connector.id]);
 
   useEffect(() => {
-    if (query) {
+    makeRequest({ searchQuery: query?.searchValue || undefined });
+    if (query?.searchValue) {
       checkIndexExists({ indexName: query.searchValue });
     }
   }, [query]);
@@ -207,9 +215,8 @@ export const AttachIndexBox: React.FC<AttachIndexBoxProps> = ({ connector }) =>
               )}
               isLoading={isLoading}
               options={groupedOptions}
-              onSearchChange={(searchValue, hasMatchingOptions) => {
+              onSearchChange={(searchValue) => {
                 setQuery({
-                  hasMatchingOptions: !!hasMatchingOptions,
                   isFullMatch: options.some((option) => option.label === searchValue),
                   searchValue,
                 });

x-pack/plugins/enterprise_search/server/lib/indices/fetch_unattached_indices.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { IScopedClusterClient } from '@kbn/core/server';
+import { fetchConnectors } from '@kbn/search-connectors';
+
+import { isNotNullish } from '../../../common/utils/is_not_nullish';
+import { fetchCrawlers } from '../crawler/fetch_crawlers';
+
+import { getUnattachedIndexData } from './utils/get_index_data';
+
+export const fetchUnattachedIndices = async (
+  client: IScopedClusterClient,
+  searchQuery: string | undefined,
+  from: number,
+  size: number
+): Promise<{
+  indexNames: string[];
+  totalResults: number;
+}> => {
+  const { indexNames } = await getUnattachedIndexData(client, searchQuery);
+  const connectors = await fetchConnectors(client.asCurrentUser, indexNames);
+  const crawlers = await fetchCrawlers(client, indexNames);
+
+  const connectedIndexNames = [
+    ...connectors.map((con) => con.index_name).filter(isNotNullish),
+    ...crawlers.map((crawler) => crawler.index_name).filter(isNotNullish),
+  ];
+
+  const indexNameSlice = indexNames
+    .filter((indexName) => !connectedIndexNames.includes(indexName))
+    .filter(isNotNullish)
+    .slice(from, from + size);
+
+  if (indexNameSlice.length === 0) {
+    return {
+      indexNames: [],
+      totalResults: indexNames.length,
+    };
+  }
+
+  return {
+    indexNames: indexNameSlice,
+    totalResults: indexNames.length,
+  };
+};

x-pack/plugins/enterprise_search/server/lib/indices/utils/get_index_data.ts

@@ -137,3 +137,33 @@ export const getIndexData = async (
     indexNames,
   };
 };
+
+export const getUnattachedIndexData = async (
+  client: IScopedClusterClient,
+  searchQuery?: string
+): Promise<{ indexData: IndicesGetResponse; indexNames: string[] }> => {
+  const expandWildcards: ExpandWildcard[] = ['open'];
+  const indexPattern = searchQuery ? `*${searchQuery}*` : '*';
+  const allIndexMatches = await client.asCurrentUser.indices.get({
+    expand_wildcards: expandWildcards,
+    // for better performance only compute aliases and settings of indices but not mappings
+    features: ['aliases', 'settings'],
+    // only get specified index properties from ES to keep the response under 536MB
+    // node.js string length limit: https://github.com/nodejs/node/issues/33960
+    filter_path: ['*.aliases', '*.settings.index.hidden', '*.settings.index.verified_before_close'],
+    index: indexPattern,
+  });
+
+  const allIndexNames = Object.keys(allIndexMatches).filter(
+    (indexName) =>
+      allIndexMatches[indexName] &&
+      !isHidden(allIndexMatches[indexName]) &&
+      !isClosed(allIndexMatches[indexName])
+  );
+  const indexNames = allIndexNames;
+
+  return {
+    indexData: allIndexMatches,
+    indexNames,
+  };
+};

x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts

@@ -35,6 +35,7 @@ import { addConnector } from '../../lib/connectors/add_connector';
 import { startSync } from '../../lib/connectors/start_sync';
 import { deleteAccessControlIndex } from '../../lib/indices/delete_access_control_index';
 import { fetchIndexCounts } from '../../lib/indices/fetch_index_counts';
+import { fetchUnattachedIndices } from '../../lib/indices/fetch_unattached_indices';
 import { generateApiKey } from '../../lib/indices/generate_api_key';
 import { deleteIndexPipelines } from '../../lib/pipelines/delete_pipelines';
 import { getDefaultPipeline } from '../../lib/pipelines/get_default_pipeline';
@@ -696,4 +697,42 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) {
       return response.ok();
     })
   );
+
+  router.get(
+    {
+      path: '/internal/enterprise_search/connectors/available_indices',
+      validate: {
+        query: schema.object({
+          from: schema.number({ defaultValue: 0, min: 0 }),
+          search_query: schema.maybe(schema.string()),
+          size: schema.number({ defaultValue: 40, min: 0 }),
+        }),
+      },
+    },
+    elasticsearchErrorHandler(log, async (context, request, response) => {
+      const { from, size, search_query: searchQuery } = request.query;
+      const { client } = (await context.core).elasticsearch;
+
+      const { indexNames, totalResults } = await fetchUnattachedIndices(
+        client,
+        searchQuery,
+        from,
+        size
+      );
+
+      return response.ok({
+        body: {
+          indexNames,
+          meta: {
+            page: {
+              from,
+              size,
+              total: totalResults,
+            },
+          },
+        },
+        headers: { 'content-type': 'application/json' },
+      });
+    })
+  );
 }