diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_all_indices_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_all_indices_api_logic.ts deleted file mode 100644 index 9d223ce3c04a1..0000000000000 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_all_indices_api_logic.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ElasticsearchIndexWithIngestion } from '../../../../../common/types/indices'; -import { Meta } from '../../../../../common/types/pagination'; - -import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic'; -import { HttpLogic } from '../../../shared/http'; - -export interface FetchAllIndicesResponse { - indices: ElasticsearchIndexWithIngestion[]; - meta: Meta; -} - -export const fetchAllIndices = async (): Promise => { - const { http } = HttpLogic.values; - const route = '/internal/enterprise_search/indices'; - const response = await http.get(route); - return response; -}; - -export const FetchAllIndicesAPILogic = createApiLogic( - ['content', 'fetch_all_indices_api_logic'], - fetchAllIndices -); - -export type FetchAllIndicesApiActions = Actions<{}, FetchAllIndicesResponse>; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_available_indices_api_logic.ts b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_available_indices_api_logic.ts new file mode 100644 index 0000000000000..4141216cd295c --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/api/index/fetch_available_indices_api_logic.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Meta } from '../../../../../common/types/pagination'; + +import { Actions, createApiLogic } from '../../../shared/api_logic/create_api_logic'; +import { INPUT_THROTTLE_DELAY_MS } from '../../../shared/constants/timers'; +import { HttpLogic } from '../../../shared/http'; + +export interface FetchAvailabeIndicesApiParams { + searchQuery?: string; +} +export interface FetchAvailableIndicesApiResponse { + indexNames: string[]; + meta: Meta; +} + +export const fetchAvailableIndices = async ({ + searchQuery, +}: FetchAvailabeIndicesApiParams): Promise => { + const { http } = HttpLogic.values; + const route = '/internal/enterprise_search/connectors/available_indices'; + const query = { search_query: searchQuery || null }; + const response = await http.get(route, { query }); + return response; +}; + +export const FetchAvailableIndicesAPILogic = createApiLogic( + ['content', 'fetch_available_indices_api_logic'], + fetchAvailableIndices, + { + requestBreakpointMS: INPUT_THROTTLE_DELAY_MS, + } +); + +export type FetchAvailableIndicesApiActions = Actions< + FetchAvailabeIndicesApiParams, + FetchAvailableIndicesApiResponse +>; diff --git a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/attach_index_box.tsx b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/attach_index_box.tsx index 34f4820bd181d..9e52ded0dcee5 100644 --- a/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/attach_index_box.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/enterprise_search_content/components/connector_detail/attach_index_box.tsx @@ -31,7 +31,7 @@ import { Connector } from '@kbn/search-connectors'; import { Status } from '../../../../../common/types/api'; -import { FetchAllIndicesAPILogic } from '../../api/index/fetch_all_indices_api_logic'; +import { FetchAvailableIndicesAPILogic } from '../../api/index/fetch_available_indices_api_logic'; import { AttachIndexLogic } from './attach_index_logic'; @@ -74,13 +74,12 @@ export const AttachIndexBox: React.FC = ({ connector }) => ); const [selectedLanguage] = useState(); const [query, setQuery] = useState<{ - hasMatchingOptions: boolean; isFullMatch: boolean; searchValue: string; }>(); - const { makeRequest } = useActions(FetchAllIndicesAPILogic); - const { data, status } = useValues(FetchAllIndicesAPILogic); + const { makeRequest } = useActions(FetchAvailableIndicesAPILogic); + const { data, status } = useValues(FetchAvailableIndicesAPILogic); const isLoading = [Status.IDLE, Status.LOADING].includes(status); const onSave = () => { @@ -93,14 +92,22 @@ export const AttachIndexBox: React.FC = ({ connector }) => const options: Array> = isLoading ? [] - : data?.indices.map((index) => { + : data?.indexNames.map((name) => { return { - label: index.name, + label: name, }; }) ?? []; - const shouldPrependUserInputAsOption = - !!query?.searchValue && query.hasMatchingOptions && !query.isFullMatch; + const hasMatchingOptions = + data?.indexNames.some((name) => + name.toLocaleLowerCase().includes(query?.searchValue.toLocaleLowerCase() ?? '') + ) ?? false; + const isFullMatch = + data?.indexNames.some( + (name) => name.toLocaleLowerCase() === query?.searchValue.toLocaleLowerCase() + ) ?? false; + + const shouldPrependUserInputAsOption = !!query?.searchValue && hasMatchingOptions && !isFullMatch; const groupedOptions: Array> = shouldPrependUserInputAsOption ? [ @@ -127,7 +134,8 @@ export const AttachIndexBox: React.FC = ({ connector }) => }, [connector.id]); useEffect(() => { - if (query) { + makeRequest({ searchQuery: query?.searchValue || undefined }); + if (query?.searchValue) { checkIndexExists({ indexName: query.searchValue }); } }, [query]); @@ -207,9 +215,8 @@ export const AttachIndexBox: React.FC = ({ connector }) => )} isLoading={isLoading} options={groupedOptions} - onSearchChange={(searchValue, hasMatchingOptions) => { + onSearchChange={(searchValue) => { setQuery({ - hasMatchingOptions: !!hasMatchingOptions, isFullMatch: options.some((option) => option.label === searchValue), searchValue, }); diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/fetch_unattached_indices.ts b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_unattached_indices.ts new file mode 100644 index 0000000000000..d5547733bcab5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/lib/indices/fetch_unattached_indices.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IScopedClusterClient } from '@kbn/core/server'; +import { fetchConnectors } from '@kbn/search-connectors'; + +import { isNotNullish } from '../../../common/utils/is_not_nullish'; +import { fetchCrawlers } from '../crawler/fetch_crawlers'; + +import { getUnattachedIndexData } from './utils/get_index_data'; + +export const fetchUnattachedIndices = async ( + client: IScopedClusterClient, + searchQuery: string | undefined, + from: number, + size: number +): Promise<{ + indexNames: string[]; + totalResults: number; +}> => { + const { indexNames } = await getUnattachedIndexData(client, searchQuery); + const connectors = await fetchConnectors(client.asCurrentUser, indexNames); + const crawlers = await fetchCrawlers(client, indexNames); + + const connectedIndexNames = [ + ...connectors.map((con) => con.index_name).filter(isNotNullish), + ...crawlers.map((crawler) => crawler.index_name).filter(isNotNullish), + ]; + + const indexNameSlice = indexNames + .filter((indexName) => !connectedIndexNames.includes(indexName)) + .filter(isNotNullish) + .slice(from, from + size); + + if (indexNameSlice.length === 0) { + return { + indexNames: [], + totalResults: indexNames.length, + }; + } + + return { + indexNames: indexNameSlice, + totalResults: indexNames.length, + }; +}; diff --git a/x-pack/plugins/enterprise_search/server/lib/indices/utils/get_index_data.ts b/x-pack/plugins/enterprise_search/server/lib/indices/utils/get_index_data.ts index 68ed32aa6d3f3..8edb60b6c8f8b 100644 --- a/x-pack/plugins/enterprise_search/server/lib/indices/utils/get_index_data.ts +++ b/x-pack/plugins/enterprise_search/server/lib/indices/utils/get_index_data.ts @@ -137,3 +137,33 @@ export const getIndexData = async ( indexNames, }; }; + +export const getUnattachedIndexData = async ( + client: IScopedClusterClient, + searchQuery?: string +): Promise<{ indexData: IndicesGetResponse; indexNames: string[] }> => { + const expandWildcards: ExpandWildcard[] = ['open']; + const indexPattern = searchQuery ? `*${searchQuery}*` : '*'; + const allIndexMatches = await client.asCurrentUser.indices.get({ + expand_wildcards: expandWildcards, + // for better performance only compute aliases and settings of indices but not mappings + features: ['aliases', 'settings'], + // only get specified index properties from ES to keep the response under 536MB + // node.js string length limit: https://github.com/nodejs/node/issues/33960 + filter_path: ['*.aliases', '*.settings.index.hidden', '*.settings.index.verified_before_close'], + index: indexPattern, + }); + + const allIndexNames = Object.keys(allIndexMatches).filter( + (indexName) => + allIndexMatches[indexName] && + !isHidden(allIndexMatches[indexName]) && + !isClosed(allIndexMatches[indexName]) + ); + const indexNames = allIndexNames; + + return { + indexData: allIndexMatches, + indexNames, + }; +}; diff --git a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts index 7c6706b5aef48..7ad04cf753dd7 100644 --- a/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts +++ b/x-pack/plugins/enterprise_search/server/routes/enterprise_search/connectors.ts @@ -35,6 +35,7 @@ import { addConnector } from '../../lib/connectors/add_connector'; import { startSync } from '../../lib/connectors/start_sync'; import { deleteAccessControlIndex } from '../../lib/indices/delete_access_control_index'; import { fetchIndexCounts } from '../../lib/indices/fetch_index_counts'; +import { fetchUnattachedIndices } from '../../lib/indices/fetch_unattached_indices'; import { generateApiKey } from '../../lib/indices/generate_api_key'; import { deleteIndexPipelines } from '../../lib/pipelines/delete_pipelines'; import { getDefaultPipeline } from '../../lib/pipelines/get_default_pipeline'; @@ -696,4 +697,42 @@ export function registerConnectorRoutes({ router, log }: RouteDependencies) { return response.ok(); }) ); + + router.get( + { + path: '/internal/enterprise_search/connectors/available_indices', + validate: { + query: schema.object({ + from: schema.number({ defaultValue: 0, min: 0 }), + search_query: schema.maybe(schema.string()), + size: schema.number({ defaultValue: 40, min: 0 }), + }), + }, + }, + elasticsearchErrorHandler(log, async (context, request, response) => { + const { from, size, search_query: searchQuery } = request.query; + const { client } = (await context.core).elasticsearch; + + const { indexNames, totalResults } = await fetchUnattachedIndices( + client, + searchQuery, + from, + size + ); + + return response.ok({ + body: { + indexNames, + meta: { + page: { + from, + size, + total: totalResults, + }, + }, + }, + headers: { 'content-type': 'application/json' }, + }); + }) + ); }